{ "benchmarks": [ { "aliases": [], "category_id": "category_professional_tasks", "id": "benchmark_aa_briefcase", "name": "AA-Briefcase", "provenance": { "record_ids": [ "benchmark_aa_briefcase" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_software_engineering", "id": "benchmark_aa_coding_agent_index", "name": "AA Coding Agent Index", "provenance": { "record_ids": [ "benchmark_aa_coding_agent_index" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_aa_lcr", "name": "AA-LCR", "provenance": { "record_ids": [ "benchmark_aa_lcr" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "AAV capsid packaging" ], "category_id": "category_life_sciences", "id": "benchmark_aav_capsid_packaging_prediction", "name": "AAV Capsid Packaging Prediction", "provenance": { "record_ids": [ "benchmark_aav_capsid_packaging_prediction" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "ABC Bench FD" ], "category_id": "category_health_and_science", "id": "benchmark_abc_bench_fragment_design", "name": "ABC Bench Fragment Design", "provenance": { "record_ids": [ "benchmark_abc_bench_fragment_design" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "ABC Bench LH" ], "category_id": "category_health_and_science", "id": "benchmark_abc_bench_liquid_handling", "name": "ABC Bench Liquid Handling", "provenance": { "record_ids": [ "benchmark_abc_bench_liquid_handling" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_science", "id": "benchmark_abc_bench_screening_evasion", "name": "ABC Bench Screening Evasion", "provenance": { "record_ids": [ "benchmark_abc_bench_screening_evasion" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_activitynet_qa", "name": "ActivityNet-QA", "provenance": { "record_ids": [ "benchmark_activitynet_qa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Adv SQuAD" ], "category_id": "category_robustness_and_safety", "id": "benchmark_adversarial_squad", "name": "Adversarial SQuAD", "provenance": { "record_ids": [ "benchmark_adversarial_squad" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Agent Red Teaming (ART) benchmark", "Gray Swan Agent Red Teaming benchmark" ], "category_id": "category_prompt_injection", "id": "benchmark_agent_red_teaming", "name": "Agent Red Teaming", "provenance": { "record_ids": [ "benchmark_agent_red_teaming" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_agentdojo", "name": "AgentDojo", "provenance": { "record_ids": [ "benchmark_agentdojo" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_agentharm", "name": "AgentHarm", "provenance": { "record_ids": [ "benchmark_agentharm" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_agentharm_verified", "name": "AgentHarm Verified", "provenance": { "record_ids": [ "benchmark_agentharm_verified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_agentic_cybermisuse", "name": "Agentic Cybermisuse", "provenance": { "record_ids": [ "benchmark_agentic_cybermisuse" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_agentic_misalignment", "name": "Agentic Misalignment", "provenance": { "record_ids": [ "benchmark_agentic_misalignment" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "agentic misalignment evaluation suite" ], "category_id": "category_agentic_misalignment", "id": "benchmark_agentic_misalignment_suite", "name": "Agentic Misalignment Suite", "provenance": { "record_ids": [ "benchmark_agentic_misalignment_suite" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Agent Last Exam" ], "category_id": "category_professional_work", "id": "benchmark_agents_last_exam", "name": "Agents' Last Exam", "provenance": { "record_ids": [ "benchmark_agents_last_exam" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reasoning_and_knowledge", "id": "benchmark_agieval", "name": "AGIEval", "provenance": { "record_ids": [ "benchmark_agieval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_agieval_english", "name": "AGIEval English", "provenance": { "record_ids": [ "benchmark_agieval_english" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "question answering about science diagrams", "science diagram benchmark", "Science diagrams", "AI2 Diagram" ], "category_id": "category_science_diagram_reasoning", "id": "benchmark_ai2d", "name": "AI2D", "provenance": { "record_ids": [ "benchmark_ai2d" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_ai2d_test", "name": "AI2D Test", "provenance": { "record_ids": [ "benchmark_ai2d_test" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_aider", "name": "Aider", "provenance": { "record_ids": [ "benchmark_aider" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_aider_edit", "name": "Aider-Edit", "provenance": { "record_ids": [ "benchmark_aider_edit" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Aider's polyglot diff benchmark", "Aider-Polyglot" ], "category_id": "category_coding", "id": "benchmark_aider_polyglot", "name": "Aider Polyglot", "provenance": { "record_ids": [ "benchmark_aider_polyglot" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_aime", "name": "AIME", "provenance": { "record_ids": [ "benchmark_aime" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "American Invitational Mathematics Examination", "American Invitational Mathematics Examination 2024" ], "category_id": "category_mathematics", "id": "benchmark_aime_2024", "name": "AIME 2024", "provenance": { "record_ids": [ "benchmark_aime_2024" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Expanded as American Invitational Mathematics Examination" ], "category_id": "category_mathematics", "id": "benchmark_aime_2025", "name": "AIME 2025", "provenance": { "record_ids": [ "benchmark_aime_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_aime_2026", "name": "AIME 2026", "provenance": { "record_ids": [ "benchmark_aime_2026" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics_and_science", "id": "benchmark_aime_year_not_specified", "name": "AIME (year not specified)", "provenance": { "record_ids": [ "benchmark_aime_year_not_specified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_tool_use_and_agents", "id": "benchmark_airs_bench", "name": "AIRS-Bench", "provenance": { "record_ids": [ "benchmark_airs_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_aitz", "name": "AITZ", "provenance": { "record_ids": [ "benchmark_aitz" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_alfred", "name": "ALFRED", "provenance": { "record_ids": [ "benchmark_alfred" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_and_preference", "id": "benchmark_alignbench", "name": "AlignBench", "provenance": { "record_ids": [ "benchmark_alignbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_human_preference", "id": "benchmark_alignbench_1_1", "name": "AlignBench 1.1", "provenance": { "record_ids": [ "benchmark_alignbench_1_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_alignment_faking", "name": "Alignment Faking", "provenance": { "record_ids": [ "benchmark_alignment_faking" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "AlpacaEval2.0" ], "category_id": "category_instruction_and_preference", "id": "benchmark_alpacaeval_2_0", "name": "AlpacaEval 2.0", "provenance": { "record_ids": [ "benchmark_alpacaeval_2_0" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_amc_2022_23", "name": "AMC 2022-23", "provenance": { "record_ids": [ "benchmark_amc_2022_23" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_amc_2023", "name": "AMC 2023", "provenance": { "record_ids": [ "benchmark_amc_2023" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "AMC 10" ], "category_id": "category_competition_mathematics", "id": "benchmark_american_mathematics_competition_10", "name": "American Mathematics Competition 10", "provenance": { "record_ids": [ "benchmark_american_mathematics_competition_10" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "AMC 12" ], "category_id": "category_competition_mathematics", "id": "benchmark_american_mathematics_competition_12", "name": "American Mathematics Competition 12", "provenance": { "record_ids": [ "benchmark_american_mathematics_competition_12" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "AMC 8" ], "category_id": "category_competition_mathematics", "id": "benchmark_american_mathematics_competition_8", "name": "American Mathematics Competition 8", "provenance": { "record_ids": [ "benchmark_american_mathematics_competition_8" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_ammlu", "name": "AMMLU", "provenance": { "record_ids": [ "benchmark_ammlu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_android_control", "name": "Android Control", "provenance": { "record_ids": [ "benchmark_android_control" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_androidbench", "name": "AndroidBench", "provenance": { "record_ids": [ "benchmark_androidbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Android World" ], "category_id": "category_agentic_and_tool_use", "id": "benchmark_androidworld", "name": "AndroidWorld", "provenance": { "record_ids": [ "benchmark_androidworld" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_art_history", "name": "AP Art History", "provenance": { "record_ids": [ "benchmark_ap_art_history" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_biology", "name": "AP Biology", "provenance": { "record_ids": [ "benchmark_ap_biology" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_calculus", "name": "AP Calculus", "provenance": { "record_ids": [ "benchmark_ap_calculus" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_chemistry", "name": "AP Chemistry", "provenance": { "record_ids": [ "benchmark_ap_chemistry" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_english_language", "name": "AP English Language", "provenance": { "record_ids": [ "benchmark_ap_english_language" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_english_literature", "name": "AP English Literature", "provenance": { "record_ids": [ "benchmark_ap_english_literature" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_environmental_science", "name": "AP Environmental Science", "provenance": { "record_ids": [ "benchmark_ap_environmental_science" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_macroeconomics", "name": "AP Macroeconomics", "provenance": { "record_ids": [ "benchmark_ap_macroeconomics" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_microeconomics", "name": "AP Microeconomics", "provenance": { "record_ids": [ "benchmark_ap_microeconomics" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_physics", "name": "AP Physics", "provenance": { "record_ids": [ "benchmark_ap_physics" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_psychology", "name": "AP Psychology", "provenance": { "record_ids": [ "benchmark_ap_psychology" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_statistics", "name": "AP Statistics", "provenance": { "record_ids": [ "benchmark_ap_statistics" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_united_states_government", "name": "AP United States Government", "provenance": { "record_ids": [ "benchmark_ap_united_states_government" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_united_states_history", "name": "AP United States History", "provenance": { "record_ids": [ "benchmark_ap_united_states_history" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_ap_world_history", "name": "AP World History", "provenance": { "record_ids": [ "benchmark_ap_world_history" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_tool_use", "id": "benchmark_apex", "name": "Apex", "provenance": { "record_ids": [ "benchmark_apex" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_work", "id": "benchmark_apex_agents", "name": "APEX-Agents", "provenance": { "record_ids": [ "benchmark_apex_agents" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_tool_use", "id": "benchmark_apex_shortlist", "name": "Apex Shortlist", "provenance": { "record_ids": [ "benchmark_apex_shortlist" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_tool_use", "id": "benchmark_apexbench", "name": "ApexBench", "provenance": { "record_ids": [ "benchmark_apexbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_tool_use_and_agents", "id": "benchmark_api_bank", "name": "API-Bank", "provenance": { "record_ids": [ "benchmark_api_bank" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_code_generation", "id": "benchmark_apps", "name": "APPS", "provenance": { "record_ids": [ "benchmark_apps" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "APPS/Intro" ], "category_id": "category_coding", "id": "benchmark_apps_intro", "name": "APPS Intro", "provenance": { "record_ids": [ "benchmark_apps_intro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "The summary table groups the family under ARC-AGI" ], "category_id": "category_mathematics_and_science", "id": "benchmark_arc_agi_1", "name": "ARC-AGI-1", "provenance": { "record_ids": [ "benchmark_arc_agi_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "The launch footnote prints ARC-AGI-2", "The summary table prints ARC-AGI-2 (Verified)", "ARCAGI 2" ], "category_id": "category_mathematics_and_science", "id": "benchmark_arc_agi_2", "name": "ARC-AGI-2", "provenance": { "record_ids": [ "benchmark_arc_agi_2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "ARC-AGI 3" ], "category_id": "category_abstract_reasoning", "id": "benchmark_arc_agi_3", "name": "ARC-AGI-3", "provenance": { "record_ids": [ "benchmark_arc_agi_3" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "AI2 Reasoning Challenge", "ARC-Challenge", "ARC-C", "AI2 Reasoning Challenge Challenge Set" ], "category_id": "category_commonsense_and_science_reasoning", "id": "benchmark_arc_c", "name": "ARC Challenge", "provenance": { "record_ids": [ "benchmark_arc_c" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "ARC-E" ], "category_id": "category_reasoning_and_knowledge", "id": "benchmark_arc_easy", "name": "ARC-Easy", "provenance": { "record_ids": [ "benchmark_arc_easy" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_human_preference", "id": "benchmark_arena", "name": "Arena", "provenance": { "record_ids": [ "benchmark_arena" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Arena.ai Leaderboard" ], "category_id": "category_reasoning_and_knowledge", "id": "benchmark_arena_ai_text_leaderboard", "name": "Arena.ai Text Leaderboard", "provenance": { "record_ids": [ "benchmark_arena_ai_text_leaderboard" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "ArenaHard" ], "category_id": "category_instruction_and_preference", "id": "benchmark_arena_hard", "name": "Arena-Hard", "provenance": { "record_ids": [ "benchmark_arena_hard" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_human_preference", "id": "benchmark_arena_hard_v2", "name": "Arena-Hard v2", "provenance": { "record_ids": [ "benchmark_arena_hard_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_arkitscenes", "name": "ARKitScenes", "provenance": { "record_ids": [ "benchmark_arkitscenes" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_work", "id": "benchmark_artificial_analysis_coding_agent_index", "name": "Artificial Analysis Coding Agent Index", "provenance": { "record_ids": [ "benchmark_artificial_analysis_coding_agent_index" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_work", "id": "benchmark_artificial_analysis_intelligence_index", "name": "Artificial Analysis Intelligence Index", "provenance": { "record_ids": [ "benchmark_artificial_analysis_intelligence_index" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematical_reasoning", "id": "benchmark_arxivmath", "name": "ArxivMath", "provenance": { "record_ids": [ "benchmark_arxivmath" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "ASROB" ], "category_id": "category_long_context_understanding", "id": "benchmark_automatic_speech_recognition_from_one_book", "name": "Automatic Speech Recognition from One Book", "provenance": { "record_ids": [ "benchmark_automatic_speech_recognition_from_one_book" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Automation Bench", "Zapier AutomationBench", "Automation-Bench" ], "category_id": "category_tool_use", "id": "benchmark_automation_bench", "name": "AutomationBench", "provenance": { "record_ids": [ "benchmark_automation_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_av_speakerbench", "name": "AV-SpeakerBench", "provenance": { "record_ids": [ "benchmark_av_speakerbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_avut", "name": "AVUT", "provenance": { "record_ids": [ "benchmark_avut" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_babyvision", "name": "BabyVision", "provenance": { "record_ids": [ "benchmark_babyvision" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "BIG-Bench-Hard", "Mixed evaluations", "BBH", "BigBench-Hard" ], "category_id": "category_mixed_reasoning", "id": "benchmark_bbh", "name": "BIG-Bench Hard", "provenance": { "record_ids": [ "benchmark_bbh" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Bias Benchmark for QA (BBQ)", "Bias Benchmark for Question Answering", "Bias Benchmark for Question Answering (BBQ)" ], "category_id": "category_safety_and_alignment", "id": "benchmark_bbq", "name": "BBQ", "provenance": { "record_ids": [ "benchmark_bbq" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "BBQ Ambig" ], "category_id": "category_safety_and_responsibility", "id": "benchmark_bbq_ambiguous", "name": "BBQ Ambiguous", "provenance": { "record_ids": [ "benchmark_bbq_ambiguous" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "BBQ Disambig" ], "category_id": "category_safety_and_responsibility", "id": "benchmark_bbq_disambiguated", "name": "BBQ Disambiguated", "provenance": { "record_ids": [ "benchmark_bbq_disambiguated" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_bc_vl", "name": "BC-VL", "provenance": { "record_ids": [ "benchmark_bc_vl" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_belebele", "name": "BELEBELE", "provenance": { "record_ids": [ "benchmark_belebele" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "TAU bench", "TAU-bench (τ-bench)", "TAU-retail", "τ-bench" ], "category_id": "category_agentic_tool_use", "id": "benchmark_bench", "name": "Tau-bench", "provenance": { "record_ids": [ "benchmark_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_computer_use", "id": "benchmark_benchcad", "name": "BenchCAD", "provenance": { "record_ids": [ "benchmark_benchcad" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "BFCL" ], "category_id": "category_tool_use_and_agents", "id": "benchmark_berkeley_function_calling_leaderboard", "name": "Berkeley Function Calling Leaderboard", "provenance": { "record_ids": [ "benchmark_berkeley_function_calling_leaderboard" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "BFCL v2" ], "category_id": "category_tool_use_and_agents", "id": "benchmark_berkeley_function_calling_leaderboard_v2", "name": "Berkeley Function Calling Leaderboard v2", "provenance": { "record_ids": [ "benchmark_berkeley_function_calling_leaderboard_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_betterchartqa", "name": "BetterChartQA", "provenance": { "record_ids": [ "benchmark_betterchartqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "BFCL_v3_MultiTurn" ], "category_id": "category_agentic_tool_use", "id": "benchmark_bfcl_v3_multiturn", "name": "BFCL v3 MultiTurn", "provenance": { "record_ids": [ "benchmark_bfcl_v3_multiturn" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_bfcl_v4", "name": "BFCL v4", "provenance": { "record_ids": [ "benchmark_bfcl_v4" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "BBEH" ], "category_id": "category_reasoning_and_knowledge", "id": "benchmark_big_bench_extra_hard", "name": "BIG-Bench Extra Hard", "provenance": { "record_ids": [ "benchmark_big_bench_extra_hard" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_work", "id": "benchmark_big_finance_bench", "name": "Big Finance Bench", "provenance": { "record_ids": [ "benchmark_big_finance_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_big_sleep_evaluation", "name": "Big Sleep Evaluation", "provenance": { "record_ids": [ "benchmark_big_sleep_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_bigcodebench", "name": "BigCodeBench", "provenance": { "record_ids": [ "benchmark_bigcodebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_work", "id": "benchmark_biglaw_bench", "name": "BigLaw Bench", "provenance": { "record_ids": [ "benchmark_biglaw_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_summarization", "id": "benchmark_bigpatent", "name": "BigPatent", "provenance": { "record_ids": [ "benchmark_bigpatent" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_biomedical_question_answering", "id": "benchmark_bioasq", "name": "BioASQ", "provenance": { "record_ids": [ "benchmark_bioasq" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_science", "id": "benchmark_biodesign_tools", "name": "BioDesign Tools", "provenance": { "record_ids": [ "benchmark_biodesign_tools" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_biolp_bench", "name": "BioLP-Bench", "provenance": { "record_ids": [ "benchmark_biolp_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_computational_biology", "id": "benchmark_biomysterybench", "name": "BioMysteryBench", "provenance": { "record_ids": [ "benchmark_biomysterybench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_life_sciences", "id": "benchmark_biomysterybench_verified", "name": "BioMysteryBench Verified", "provenance": { "record_ids": [ "benchmark_biomysterybench_verified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_computational_biology", "id": "benchmark_biopipelinebench", "name": "BioPipelineBench", "provenance": { "record_ids": [ "benchmark_biopipelinebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_life_sciences", "id": "benchmark_biopipelinebench_verified", "name": "BioPipelineBench Verified", "provenance": { "record_ids": [ "benchmark_biopipelinebench_verified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_biotier_refusal", "name": "BioTIER Refusal", "provenance": { "record_ids": [ "benchmark_biotier_refusal" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Bioweapons Knowledge Questions Evaluation" ], "category_id": "category_biological_risk_knowledge", "id": "benchmark_bioweapons_knowledge_questions", "name": "Bioweapons Knowledge Questions", "provenance": { "record_ids": [ "benchmark_bioweapons_knowledge_questions" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_bird_sql", "name": "BIRD-SQL", "provenance": { "record_ids": [ "benchmark_bird_sql" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_bixbench", "name": "BixBench", "provenance": { "record_ids": [ "benchmark_bixbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Black-box RNA sequence design" ], "category_id": "category_life_sciences", "id": "benchmark_black_box_rna_sequence_modeling_and_design", "name": "Black-box RNA Sequence Modeling and Design", "provenance": { "record_ids": [ "benchmark_black_box_rna_sequence_modeling_and_design" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_interpretability", "id": "benchmark_blackmail_suite", "name": "Blackmail Suite", "provenance": { "record_ids": [ "benchmark_blackmail_suite" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_blend", "name": "BLEnD", "provenance": { "record_ids": [ "benchmark_blend" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_blink", "name": "BLINK", "provenance": { "record_ids": [ "benchmark_blink" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal", "id": "benchmark_blueprint_bench_2", "name": "Blueprint-Bench 2", "provenance": { "record_ids": [ "benchmark_blueprint_bench_2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_responsibility", "id": "benchmark_bold", "name": "BOLD", "provenance": { "record_ids": [ "benchmark_bold" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_question_answering", "id": "benchmark_boolq", "name": "BoolQ", "provenance": { "record_ids": [ "benchmark_boolq" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "multi agent", "Multi-Agent BrowseComp", "single agent" ], "category_id": "category_general_reasoning", "id": "benchmark_browsecomp", "name": "BrowseComp", "provenance": { "record_ids": [ "benchmark_browsecomp" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_browsecomp_long_context", "name": "BrowseComp Long Context", "provenance": { "record_ids": [ "benchmark_browsecomp_long_context" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Described as derived from BrowseComp", "The launch methodology calls one configuration a fetch-enabled version of BrowseComp-Plus" ], "category_id": "category_agentic_search", "id": "benchmark_browsecomp_plus", "name": "BrowseComp-Plus", "provenance": { "record_ids": [ "benchmark_browsecomp_plus" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "BrowseComp-zh", "BrowseComp_zh", "BrowseCompZh" ], "category_id": "category_agentic_tool_use", "id": "benchmark_browsecomp_zh", "name": "BrowseComp zh", "provenance": { "record_ids": [ "benchmark_browsecomp_zh" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reading_comprehension", "id": "benchmark_c3", "name": "C3", "provenance": { "record_ids": [ "benchmark_c3" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_text_quality", "id": "benchmark_c4", "name": "C4", "provenance": { "record_ids": [ "benchmark_c4" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_reasoning", "id": "benchmark_c_eval", "name": "C-Eval", "provenance": { "record_ids": [ "benchmark_c_eval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_factuality", "id": "benchmark_c_simpleqa", "name": "C-SimpleQA", "provenance": { "record_ids": [ "benchmark_c_simpleqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_capture_the_flag_challenges", "name": "Capture-the-Flag Challenges", "provenance": { "record_ids": [ "benchmark_capture_the_flag_challenges" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_catastrophic_cyber_misuse", "name": "Catastrophic Cyber Misuse", "provenance": { "record_ids": [ "benchmark_catastrophic_cyber_misuse" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_cc_ocr", "name": "CC-OCR", "provenance": { "record_ids": [ "benchmark_cc_ocr" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_cc_ocr_bench_v2", "name": "CC-OCR-Bench-V2", "provenance": { "record_ids": [ "benchmark_cc_ocr_bench_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_and_preference", "id": "benchmark_ccpm", "name": "CCPM", "provenance": { "record_ids": [ "benchmark_ccpm" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_cg_bench", "name": "CG-Bench", "provenance": { "record_ids": [ "benchmark_cg_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_alignment", "id": "benchmark_challenging_refusal_evaluation", "name": "Challenging Refusal Evaluation", "provenance": { "record_ids": [ "benchmark_challenging_refusal_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_alignment", "id": "benchmark_changemyview", "name": "ChangeMyView", "provenance": { "record_ids": [ "benchmark_changemyview" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_charades_sta", "name": "Charades-STA", "provenance": { "record_ids": [ "benchmark_charades_sta" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_charadessta", "name": "CharadesSTA", "provenance": { "record_ids": [ "benchmark_charadessta" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_chartmimic", "name": "ChartMimic", "provenance": { "record_ids": [ "benchmark_chartmimic" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal", "id": "benchmark_chartmuseum", "name": "ChartMuseum", "provenance": { "record_ids": [ "benchmark_chartmuseum" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal", "id": "benchmark_chartography", "name": "Chartography", "provenance": { "record_ids": [ "benchmark_chartography" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Chart understanding", "question answering about charts and graphs" ], "category_id": "category_chart_question_answering", "id": "benchmark_chartqa", "name": "ChartQA", "provenance": { "record_ids": [ "benchmark_chartqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal", "id": "benchmark_chartqapro", "name": "ChartQAPro", "provenance": { "record_ids": [ "benchmark_chartqapro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "CharXiv-D" ], "category_id": "category_multimodal_understanding", "id": "benchmark_charxiv_descriptive", "name": "CharXiv Descriptive", "provenance": { "record_ids": [ "benchmark_charxiv_descriptive" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "CharXiv-R", "Described as a CharXiv chart-understanding evaluation suite" ], "category_id": "category_multimodal_understanding", "id": "benchmark_charxiv_reasoning", "name": "CharXiv Reasoning", "provenance": { "record_ids": [ "benchmark_charxiv_reasoning" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_charxiv_rq", "name": "CharXiv RQ", "provenance": { "record_ids": [ "benchmark_charxiv_rq" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_chemical_agents_refusal", "name": "Chemical Agents Refusal", "provenance": { "record_ids": [ "benchmark_chemical_agents_refusal" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_chemicaldiagramqa", "name": "ChemicalDiagramQA", "provenance": { "record_ids": [ "benchmark_chemicaldiagramqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_factuality", "id": "benchmark_chinese_simpleqa", "name": "Chinese-SimpleQA", "provenance": { "record_ids": [ "benchmark_chinese_simpleqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_chrome_production_commit_scanning_pipeline", "name": "Chrome Production Commit Scanning Pipeline", "provenance": { "record_ids": [ "benchmark_chrome_production_commit_scanning_pipeline" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_claweval", "name": "ClawEval", "provenance": { "record_ids": [ "benchmark_claweval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_claweval_mm", "name": "ClawEval-MM", "provenance": { "record_ids": [ "benchmark_claweval_mm" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_cloningqa", "name": "CloningQA", "provenance": { "record_ids": [ "benchmark_cloningqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_biological_knowledge", "id": "benchmark_cloningscenarios", "name": "CloningScenarios", "provenance": { "record_ids": [ "benchmark_cloningscenarios" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reasoning", "id": "benchmark_cluewsc", "name": "CLUEWSC", "provenance": { "record_ids": [ "benchmark_cluewsc" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_cmath", "name": "CMath", "provenance": { "record_ids": [ "benchmark_cmath" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_reasoning", "id": "benchmark_cmmlu", "name": "CMMLU", "provenance": { "record_ids": [ "benchmark_cmmlu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_cmo_2025", "name": "CMO 2025", "provenance": { "record_ids": [ "benchmark_cmo_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reading_comprehension", "id": "benchmark_cmrc", "name": "CMRC", "provenance": { "record_ids": [ "benchmark_cmrc" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_cmt_benchmark", "name": "CMT-Benchmark", "provenance": { "record_ids": [ "benchmark_cmt_benchmark" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_cn_middle_school_2024", "name": "CN Middle School 2024", "provenance": { "record_ids": [ "benchmark_cn_middle_school_2024" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_cnmo_2024", "name": "CNMO 2024", "provenance": { "record_ids": [ "benchmark_cnmo_2024" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "CNN DailyMail", "CNN-DM" ], "category_id": "category_summarization", "id": "benchmark_cnn_dailymail", "name": "CNN/DailyMail", "provenance": { "record_ids": [ "benchmark_cnn_dailymail" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_coco", "name": "COCO", "provenance": { "record_ids": [ "benchmark_coco" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_coco_caption", "name": "COCO Caption", "provenance": { "record_ids": [ "benchmark_coco_caption" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_code_arena", "name": "Code Arena", "provenance": { "record_ids": [ "benchmark_code_arena" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_codecontests", "name": "CodeContests", "provenance": { "record_ids": [ "benchmark_codecontests" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Codeforces-Div1" ], "category_id": "category_coding", "id": "benchmark_codeforces", "name": "Codeforces", "provenance": { "record_ids": [ "benchmark_codeforces" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_codeu", "name": "CodeU", "provenance": { "record_ids": [ "benchmark_codeu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_collegemath", "name": "CollegeMath", "provenance": { "record_ids": [ "benchmark_collegemath" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_collegiate_ctfs", "name": "Collegiate CTFs", "provenance": { "record_ids": [ "benchmark_collegiate_ctfs" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_collie", "name": "COLLIE", "provenance": { "record_ids": [ "benchmark_collie" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_common_voice", "name": "Common Voice", "provenance": { "record_ids": [ "benchmark_common_voice" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_common_voice_15", "name": "Common Voice 15", "provenance": { "record_ids": [ "benchmark_common_voice_15" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "CQA", "CSQA" ], "category_id": "category_reasoning_and_knowledge", "id": "benchmark_commonsenseqa", "name": "CommonsenseQA", "provenance": { "record_ids": [ "benchmark_commonsenseqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_tool_use", "id": "benchmark_complexfuncbench", "name": "ComplexFuncBench", "provenance": { "record_ids": [ "benchmark_complexfuncbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Computer use evaluation" ], "category_id": "category_prompt_injection", "id": "benchmark_computer_use_prompt_injection_evaluation", "name": "Computer-Use Prompt Injection Evaluation", "provenance": { "record_ids": [ "benchmark_computer_use_prompt_injection_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_commonsense_reasoning", "id": "benchmark_copa", "name": "COPA", "provenance": { "record_ids": [ "benchmark_copa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_corpusqa_1m", "name": "CorpusQA 1M", "provenance": { "record_ids": [ "benchmark_corpusqa_1m" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_text_quality", "id": "benchmark_cosmopedia_coherence", "name": "Cosmopedia Coherence", "provenance": { "record_ids": [ "benchmark_cosmopedia_coherence" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_alignment", "id": "benchmark_cot_control", "name": "CoT-Control", "provenance": { "record_ids": [ "benchmark_cot_control" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_countbench", "name": "CountBench", "provenance": { "record_ids": [ "benchmark_countbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "CountBenchQA" ], "category_id": "category_multimodal_understanding", "id": "benchmark_countbench_vqa", "name": "CountBench VQA", "provenance": { "record_ids": [ "benchmark_countbench_vqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_countqa", "name": "CountQA", "provenance": { "record_ids": [ "benchmark_countqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_coursera", "name": "Coursera", "provenance": { "record_ids": [ "benchmark_coursera" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_covost2", "name": "CoVoST2", "provenance": { "record_ids": [ "benchmark_covost2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_coworkbench", "name": "CoWorkBench", "provenance": { "record_ids": [ "benchmark_coworkbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Creative biology tasks" ], "category_id": "category_biological_risk", "id": "benchmark_creative_biology", "name": "Creative Biology", "provenance": { "record_ids": [ "benchmark_creative_biology" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_human_preference", "id": "benchmark_creative_writing_v3", "name": "Creative Writing v3", "provenance": { "record_ids": [ "benchmark_creative_writing_v3" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematical_reasoning", "id": "benchmark_critpt", "name": "CritPt", "provenance": { "record_ids": [ "benchmark_critpt" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_crosscodeeval", "name": "CrossCodeEval", "provenance": { "record_ids": [ "benchmark_crosscodeeval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_crosscodelongeval", "name": "CrossCodeLongEval", "provenance": { "record_ids": [ "benchmark_crosscodelongeval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_responsibility", "id": "benchmark_crows_pairs", "name": "CrowS-Pairs", "provenance": { "record_ids": [ "benchmark_crows_pairs" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_crux_i", "name": "CRUX-I", "provenance": { "record_ids": [ "benchmark_crux_i" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_crux_o", "name": "CRUX-O", "provenance": { "record_ids": [ "benchmark_crux_o" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_cruxeval", "name": "CRUXEval", "provenance": { "record_ids": [ "benchmark_cruxeval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_cruxeval_i", "name": "CRUXEval-I", "provenance": { "record_ids": [ "benchmark_cruxeval_i" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_cruxeval_o", "name": "CRUXEval-O", "provenance": { "record_ids": [ "benchmark_cruxeval_o" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_csimpleqa", "name": "CSimpleQA", "provenance": { "record_ids": [ "benchmark_csimpleqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_curated_ctfs", "name": "Curated CTFs", "provenance": { "record_ids": [ "benchmark_curated_ctfs" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_software_engineering", "id": "benchmark_cursorbench", "name": "CursorBench", "provenance": { "record_ids": [ "benchmark_cursorbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_software_engineering", "id": "benchmark_cursorbench_3_2", "name": "CursorBench 3.2", "provenance": { "record_ids": [ "benchmark_cursorbench_3_2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_cute", "name": "CUTE", "provenance": { "record_ids": [ "benchmark_cute" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_cv_bench", "name": "CV-Bench", "provenance": { "record_ids": [ "benchmark_cv_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Cybench test suite", "public cyber capabilities benchmark" ], "category_id": "category_cybersecurity", "id": "benchmark_cybench", "name": "Cybench", "provenance": { "record_ids": [ "benchmark_cybench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_cyber_benign_use", "name": "Cyber Benign Use", "provenance": { "record_ids": [ "benchmark_cyber_benign_use" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "cyber ranges" ], "category_id": "category_autonomous_cybersecurity", "id": "benchmark_cyber_harness_network_challenges", "name": "Cyber-harness network challenges", "provenance": { "record_ids": [ "benchmark_cyber_harness_network_challenges" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_cyber_misuse", "name": "Cyber Misuse", "provenance": { "record_ids": [ "benchmark_cyber_misuse" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_cyber_misuse_chat", "name": "Cyber Misuse Chat", "provenance": { "record_ids": [ "benchmark_cyber_misuse_chat" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_cyberbench", "name": "CyberBench", "provenance": { "record_ids": [ "benchmark_cyberbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "The text also calls the task collection the Cybergym suite" ], "category_id": "category_cybersecurity", "id": "benchmark_cybergym", "name": "CyberGym", "provenance": { "record_ids": [ "benchmark_cybergym" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_cyberseceval_code_interpreter_abuse", "name": "CyberSecEval Code Interpreter Abuse", "provenance": { "record_ids": [ "benchmark_cyberseceval_code_interpreter_abuse" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_cyberseceval_insecure_coding", "name": "CyberSecEval Insecure Coding", "provenance": { "record_ids": [ "benchmark_cyberseceval_insecure_coding" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_cyberseceval_text_prompt_injection", "name": "CyberSecEval Text Prompt Injection", "provenance": { "record_ids": [ "benchmark_cyberseceval_text_prompt_injection" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_cyberseceval_vulnerability_identification", "name": "CyberSecEval Vulnerability Identification", "provenance": { "record_ids": [ "benchmark_cyberseceval_vulnerability_identification" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_cybersecurity_ctf", "name": "Cybersecurity CTF", "provenance": { "record_ids": [ "benchmark_cybersecurity_ctf" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_cyscenariobench", "name": "CyScenarioBench", "provenance": { "record_ids": [ "benchmark_cyscenariobench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_dailyomni", "name": "DailyOmni", "provenance": { "record_ids": [ "benchmark_dailyomni" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_deceptionbench", "name": "DeceptionBench", "provenance": { "record_ids": [ "benchmark_deceptionbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_deep_planning", "name": "Deep-Planning", "provenance": { "record_ids": [ "benchmark_deep_planning" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Deep Research Accuracy, Completeness, and Objectivity (DRACO)" ], "category_id": "category_agentic_search", "id": "benchmark_deep_research_accuracy_completeness_and_objectivity", "name": "Deep Research Accuracy, Completeness, and Objectivity", "provenance": { "record_ids": [ "benchmark_deep_research_accuracy_completeness_and_objectivity" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_search", "id": "benchmark_deepsearchqa", "name": "DeepSearchQA", "provenance": { "record_ids": [ "benchmark_deepsearchqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_deepswe", "name": "DeepSWE", "provenance": { "record_ids": [ "benchmark_deepswe" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_deepswe_1_1", "name": "DeepSWE 1.1", "provenance": { "record_ids": [ "benchmark_deepswe_1_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_deepswe_v1_1", "name": "DeepSWE v1.1", "provenance": { "record_ids": [ "benchmark_deepswe_v1_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_dense200", "name": "Dense200", "provenance": { "record_ids": [ "benchmark_dense200" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_design2code", "name": "Design2Code", "provenance": { "record_ids": [ "benchmark_design2code" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_biological_risk", "id": "benchmark_dna_synthesis_screening_evasion", "name": "DNA Synthesis Screening Evasion", "provenance": { "record_ids": [ "benchmark_dna_synthesis_screening_evasion" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Document understanding" ], "category_id": "category_document_visual_question_answering", "id": "benchmark_docvqa", "name": "DocVQA", "provenance": { "record_ids": [ "benchmark_docvqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_docvqa_test", "name": "DocVQA Test", "provenance": { "record_ids": [ "benchmark_docvqa_test" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Expanded as Deep Research Accuracy, Completeness, and Objectivity" ], "category_id": "category_deep_research", "id": "benchmark_draco", "name": "DRACO", "provenance": { "record_ids": [ "benchmark_draco" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_dream_1k", "name": "DREAM-1K", "provenance": { "record_ids": [ "benchmark_dream_1k" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Reading comprehension, arithmetic" ], "category_id": "category_reading_comprehension_and_discrete_reasoning", "id": "benchmark_drop", "name": "DROP", "provenance": { "record_ids": [ "benchmark_drop" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_ds_arena_code", "name": "DS-Arena-Code", "provenance": { "record_ids": [ "benchmark_ds_arena_code" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_ds_fim_eval", "name": "DS-FIM-Eval", "provenance": { "record_ids": [ "benchmark_ds_fim_eval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_dsbench_fullstack", "name": "DSBench-FullStack", "provenance": { "record_ids": [ "benchmark_dsbench_fullstack" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_dsbench_hard", "name": "DSBench-Hard", "provenance": { "record_ids": [ "benchmark_dsbench_hard" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_dude", "name": "DUDE", "provenance": { "record_ids": [ "benchmark_dude" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_dynabench_squad", "name": "Dynabench SQuAD", "provenance": { "record_ids": [ "benchmark_dynabench_squad" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_dynamath", "name": "DynaMath", "provenance": { "record_ids": [ "benchmark_dynamath" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_internal_factuality", "id": "benchmark_easy_medium_qa", "name": "Easy-Medium QA", "provenance": { "record_ids": [ "benchmark_easy_medium_qa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_eclektic", "name": "ECLeKTic", "provenance": { "record_ids": [ "benchmark_eclektic" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_ego3d_bench", "name": "Ego3D-Bench", "provenance": { "record_ids": [ "benchmark_ego3d_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_egolife_w_mem", "name": "EgoLife (w/ Mem.)", "provenance": { "record_ids": [ "benchmark_egolife_w_mem" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_egoschema", "name": "EgoSchema", "provenance": { "record_ids": [ "benchmark_egoschema" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_embspatialbench", "name": "EmbSpatialBench", "provenance": { "record_ids": [ "benchmark_embspatialbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_erqa", "name": "ERQA", "provenance": { "record_ids": [ "benchmark_erqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_evalplus", "name": "EvalPlus", "provenance": { "record_ids": [ "benchmark_evalplus" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_eventhallusion", "name": "EventHallusion", "provenance": { "record_ids": [ "benchmark_eventhallusion" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_expert_swe", "name": "Expert-SWE", "provenance": { "record_ids": [ "benchmark_expert_swe" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Exploit Bench" ], "category_id": "category_cybersecurity", "id": "benchmark_exploitbench", "name": "ExploitBench", "provenance": { "record_ids": [ "benchmark_exploitbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_exploitgym", "name": "ExploitGym", "provenance": { "record_ids": [ "benchmark_exploitgym" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_factuality", "id": "benchmark_facts_grounding", "name": "FACTS Grounding", "provenance": { "record_ids": [ "benchmark_facts_grounding" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_factuality", "id": "benchmark_facts_parametric", "name": "FACTS Parametric", "provenance": { "record_ids": [ "benchmark_facts_parametric" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_factuality", "id": "benchmark_factscore", "name": "FActScore", "provenance": { "record_ids": [ "benchmark_factscore" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_honesty", "id": "benchmark_false_premise_questions", "name": "False-Premise Questions", "provenance": { "record_ids": [ "benchmark_false_premise_questions" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_biological_knowledge", "id": "benchmark_figqa", "name": "FigQA", "provenance": { "record_ids": [ "benchmark_figqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Earlier card prints FinanceAgent", "Finance Agent", "Later cards print Finance Agent with a space" ], "category_id": "category_professional_work", "id": "benchmark_finance_agent", "name": "FinanceAgent", "provenance": { "record_ids": [ "benchmark_finance_agent" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_tasks", "id": "benchmark_finance_agent_v2", "name": "Finance Agent v2", "provenance": { "record_ids": [ "benchmark_finance_agent_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_interpretability", "id": "benchmark_financial_crimes", "name": "Financial Crimes", "provenance": { "record_ids": [ "benchmark_financial_crimes" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_finance", "id": "benchmark_financial_services_benchmark", "name": "Financial Services Benchmark", "provenance": { "record_ids": [ "benchmark_financial_services_benchmark" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Firefox" ], "category_id": "category_cybersecurity", "id": "benchmark_firefox_147_exploit_evaluation", "name": "Firefox 147 Exploit Evaluation", "provenance": { "record_ids": [ "benchmark_firefox_147_exploit_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Figure 3.3.3.A calls it the Firefox shell exploitation evaluation" ], "category_id": "category_cybersecurity", "id": "benchmark_firefox_147_exploitation_evaluation", "name": "Firefox 147 Exploitation Evaluation", "provenance": { "record_ids": [ "benchmark_firefox_147_exploitation_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Firefox exploits" ], "category_id": "category_cybersecurity", "id": "benchmark_firefox_exploit_evaluation", "name": "Firefox Exploit Evaluation", "provenance": { "record_ids": [ "benchmark_firefox_exploit_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_fleurs", "name": "FLEURS", "provenance": { "record_ids": [ "benchmark_fleurs" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_flickr30k_captions", "name": "Flickr30k Captions", "provenance": { "record_ids": [ "benchmark_flickr30k_captions" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_flores", "name": "FLoRes", "provenance": { "record_ids": [ "benchmark_flores" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_flores_101", "name": "FLORES-101", "provenance": { "record_ids": [ "benchmark_flores_101" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_fncall", "name": "FnCall", "provenance": { "record_ids": [ "benchmark_fncall" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_fortress", "name": "FORTRESS", "provenance": { "record_ids": [ "benchmark_fortress" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_factuality", "id": "benchmark_frames", "name": "FRAMES", "provenance": { "record_ids": [ "benchmark_frames" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "FrontierBench v0.1" ], "category_id": "category_software_engineering", "id": "benchmark_frontier_bench_v0_1", "name": "Frontier-Bench v0.1", "provenance": { "record_ids": [ "benchmark_frontier_bench_v0_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics_and_science", "id": "benchmark_frontier_science_research", "name": "Frontier Science Research", "provenance": { "record_ids": [ "benchmark_frontier_science_research" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Cognition's frontier coding eval" ], "category_id": "category_software_engineering", "id": "benchmark_frontierbench", "name": "FrontierBench", "provenance": { "record_ids": [ "benchmark_frontierbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "FrontierCode Diamond" ], "category_id": "category_software_engineering", "id": "benchmark_frontiercode", "name": "FrontierCode", "provenance": { "record_ids": [ "benchmark_frontiercode" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_software_engineering", "id": "benchmark_frontiercode_v1", "name": "FrontierCode v1", "provenance": { "record_ids": [ "benchmark_frontiercode_v1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "FrontierCode (Extended)", "FrontierCode (Main)", "FrontierCode 1.1" ], "category_id": "category_software_engineering", "id": "benchmark_frontiercode_v1_1", "name": "FrontierCode v1.1", "provenance": { "record_ids": [ "benchmark_frontiercode_v1_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_frontiercode_v2_main", "name": "FrontierCode v2 Main", "provenance": { "record_ids": [ "benchmark_frontiercode_v2_main" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics_and_science", "id": "benchmark_frontiermath", "name": "FrontierMath", "provenance": { "record_ids": [ "benchmark_frontiermath" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics_and_science", "id": "benchmark_frontiermath_v2", "name": "FrontierMath v2", "provenance": { "record_ids": [ "benchmark_frontiermath_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_science", "id": "benchmark_frontierscience_research", "name": "FrontierScience Research", "provenance": { "record_ids": [ "benchmark_frontierscience_research" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_software_engineering", "id": "benchmark_frontierswe", "name": "FrontierSWE", "provenance": { "record_ids": [ "benchmark_frontierswe" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_functional_math", "name": "Functional MATH", "provenance": { "record_ids": [ "benchmark_functional_math" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_gaokao", "name": "GaoKao", "provenance": { "record_ids": [ "benchmark_gaokao" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_gaokao_2023_english", "name": "GaoKao 2023 English", "provenance": { "record_ids": [ "benchmark_gaokao_2023_english" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_gaokao_math_cloze", "name": "GaoKao Math Cloze", "provenance": { "record_ids": [ "benchmark_gaokao_math_cloze" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_gaokao_math_qa", "name": "GaoKao Math QA", "provenance": { "record_ids": [ "benchmark_gaokao_math_qa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context_understanding", "id": "benchmark_gdm_mrcr_v2", "name": "GDM-MRCR v2", "provenance": { "record_ids": [ "benchmark_gdm_mrcr_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_gdm_situational_awareness", "name": "GDM Situational Awareness", "provenance": { "record_ids": [ "benchmark_gdm_situational_awareness" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_gdm_stealth", "name": "GDM-Stealth", "provenance": { "record_ids": [ "benchmark_gdm_stealth" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal", "id": "benchmark_gdp_pdf", "name": "GDP.pdf", "provenance": { "record_ids": [ "benchmark_gdp_pdf" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_work", "id": "benchmark_gdpval", "name": "GDPval", "provenance": { "record_ids": [ "benchmark_gdpval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Uses tasks from the GDPval gold database", "Uses tasks from the GDPval gold dataset" ], "category_id": "category_professional_tasks", "id": "benchmark_gdpval_aa", "name": "GDPval-AA", "provenance": { "record_ids": [ "benchmark_gdpval_aa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_work", "id": "benchmark_gdpval_aa_v2", "name": "GDPval-AA v2", "provenance": { "record_ids": [ "benchmark_gdpval_aa_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_genebench", "name": "GeneBench", "provenance": { "record_ids": [ "benchmark_genebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_genebench_pro", "name": "GeneBench Pro", "provenance": { "record_ids": [ "benchmark_genebench_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Abbreviated as GMMLU", "Global MMLU (GMMLU)", "GMMLU" ], "category_id": "category_multilingual", "id": "benchmark_global_mmlu", "name": "Global MMLU", "provenance": { "record_ids": [ "benchmark_global_mmlu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Global-MMLU-Lite", "GMMLU-Lite" ], "category_id": "category_multilingual", "id": "benchmark_global_mmlu_lite", "name": "Global MMLU-Lite", "provenance": { "record_ids": [ "benchmark_global_mmlu_lite" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_global_piqa", "name": "Global PIQA", "provenance": { "record_ids": [ "benchmark_global_piqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_gmat_quantitative", "name": "GMAT Quantitative", "provenance": { "record_ids": [ "benchmark_gmat_quantitative" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_gmat_verbal", "name": "GMAT Verbal", "provenance": { "record_ids": [ "benchmark_gmat_verbal" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "API-Bench" ], "category_id": "category_tool_use_and_agents", "id": "benchmark_gorilla_api_bench", "name": "Gorilla API-Bench", "provenance": { "record_ids": [ "benchmark_gorilla_api_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Graduate-Level Google-Proof Q&A" ], "category_id": "category_mathematics_and_science", "id": "benchmark_gpqa", "name": "GPQA", "provenance": { "record_ids": [ "benchmark_gpqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_gpqa_biology", "name": "GPQA Biology", "provenance": { "record_ids": [ "benchmark_gpqa_biology" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Diamond set", "GPQA (Diamond)", "GPQA is expanded as Graduate-Level Google-Proof Q&A", "Graduate level Q&A", "Graduate-Level Google-Proof Q&A Diamond", "GPQA-Diamond" ], "category_id": "category_mathematics_and_science", "id": "benchmark_gpqa_diamond", "name": "GPQA Diamond", "provenance": { "record_ids": [ "benchmark_gpqa_diamond" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematical_reasoning", "id": "benchmark_gpqa_main", "name": "GPQA Main", "provenance": { "record_ids": [ "benchmark_gpqa_main" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Graphwalks BFS", "Reported with BFS and parent-node variants", "Reported with BFS and Parents task variants" ], "category_id": "category_long_context", "id": "benchmark_graphwalks", "name": "GraphWalks", "provenance": { "record_ids": [ "benchmark_graphwalks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_graphwalks_bfs_256k", "name": "GraphWalks BFS 256K", "provenance": { "record_ids": [ "benchmark_graphwalks_bfs_256k" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_graphwalks_parents_256k", "name": "GraphWalks Parents 256K", "provenance": { "record_ids": [ "benchmark_graphwalks_parents_256k" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_grayswan_art", "name": "GraySwan ART", "provenance": { "record_ids": [ "benchmark_grayswan_art" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "GRE (Writing)" ], "category_id": "category_standardized_analytical_writing", "id": "benchmark_gre_general_test_analytical_writing", "name": "GRE General Test Analytical Writing", "provenance": { "record_ids": [ "benchmark_gre_general_test_analytical_writing" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "GRE (Quantitative)", "GRE Quantitative" ], "category_id": "category_standardized_quantitative_reasoning", "id": "benchmark_gre_general_test_quantitative_reasoning", "name": "GRE General Test Quantitative Reasoning", "provenance": { "record_ids": [ "benchmark_gre_general_test_quantitative_reasoning" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "GRE (Verbal)", "GRE Verbal" ], "category_id": "category_standardized_verbal_reasoning", "id": "benchmark_gre_general_test_verbal_reasoning", "name": "GRE General Test Verbal Reasoning", "provenance": { "record_ids": [ "benchmark_gre_general_test_verbal_reasoning" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_gre_physics", "name": "GRE Physics", "provenance": { "record_ids": [ "benchmark_gre_physics" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_gsm", "name": "GSM", "provenance": { "record_ids": [ "benchmark_gsm" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Grade school math" ], "category_id": "category_mathematics", "id": "benchmark_gsm8k", "name": "GSM8K", "provenance": { "record_ids": [ "benchmark_gsm8k" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_gsm_plus", "name": "GSM-Plus", "provenance": { "record_ids": [ "benchmark_gsm_plus" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_gtzan", "name": "GTZAN", "provenance": { "record_ids": [ "benchmark_gtzan" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_gym_cards", "name": "Gym Cards", "provenance": { "record_ids": [ "benchmark_gym_cards" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_hallbench", "name": "HallBench", "provenance": { "record_ids": [ "benchmark_hallbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_hallusionbench", "name": "HallusionBench", "provenance": { "record_ids": [ "benchmark_hallusionbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_healthcare", "id": "benchmark_healthadminbench", "name": "HealthAdminBench", "provenance": { "record_ids": [ "benchmark_healthadminbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_healthbench", "name": "HealthBench", "provenance": { "record_ids": [ "benchmark_healthbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_science", "id": "benchmark_healthbench_hard", "name": "HealthBench Hard", "provenance": { "record_ids": [ "benchmark_healthbench_hard" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_science", "id": "benchmark_healthbench_pro", "name": "HealthBench Pro", "provenance": { "record_ids": [ "benchmark_healthbench_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "HealthBench Professional adjusted" ], "category_id": "category_health_and_biology", "id": "benchmark_healthbench_professional", "name": "HealthBench Professional", "provenance": { "record_ids": [ "benchmark_healthbench_professional" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Hebbia’s Finance Benchmark" ], "category_id": "category_professional_tasks", "id": "benchmark_hebbia_finance_benchmark", "name": "Hebbia Finance Benchmark", "provenance": { "record_ids": [ "benchmark_hebbia_finance_benchmark" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_commonsense_reasoning", "id": "benchmark_hellaswag", "name": "HellaSwag", "provenance": { "record_ids": [ "benchmark_hellaswag" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_hellaswag_noise", "name": "HellaSwag Noise", "provenance": { "record_ids": [ "benchmark_hellaswag_noise" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_hiddenmath", "name": "HiddenMath", "provenance": { "record_ids": [ "benchmark_hiddenmath" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_high_school_ctfs", "name": "High School CTFs", "provenance": { "record_ids": [ "benchmark_high_school_ctfs" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_hipho", "name": "HiPhO", "provenance": { "record_ids": [ "benchmark_hipho" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "HLE", "Abbreviated as HLE", "Humanity’s Last Exam", "Humanity’s Last Exam (HLE)", "Later cards abbreviate the benchmark as HLE", "The launch page abbreviates it as HLE in an update note", "The summary table abbreviates the benchmark as HLE", "Human Last Exam", "Humanity's Last Exam with tools" ], "category_id": "category_general_reasoning", "id": "benchmark_hle", "name": "Humanity's Last Exam", "provenance": { "record_ids": [ "benchmark_hle" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_hle_calibration", "name": "HLE Calibration", "provenance": { "record_ids": [ "benchmark_hle_calibration" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_hle_verified", "name": "HLE-Verified", "provenance": { "record_ids": [ "benchmark_hle_verified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_hle_vl_w_tools", "name": "HLE-VL (w/ Tools)", "provenance": { "record_ids": [ "benchmark_hle_vl_w_tools" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_hle_with_tools", "name": "HLE with tools", "provenance": { "record_ids": [ "benchmark_hle_with_tools" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_hmmt_2025", "name": "HMMT 2025", "provenance": { "record_ids": [ "benchmark_hmmt_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_hmmt_2026_feb", "name": "HMMT 2026 Feb", "provenance": { "record_ids": [ "benchmark_hmmt_2026_feb" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_hmmt_feb_2025", "name": "HMMT Feb 2025", "provenance": { "record_ids": [ "benchmark_hmmt_feb_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics_and_science", "id": "benchmark_hmmt_february_2025", "name": "HMMT February 2025", "provenance": { "record_ids": [ "benchmark_hmmt_february_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_hmmt_february_2026", "name": "HMMT February 2026", "provenance": { "record_ids": [ "benchmark_hmmt_february_2026" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_hmmt_nov_2025", "name": "HMMT Nov 2025", "provenance": { "record_ids": [ "benchmark_hmmt_nov_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_hmmt_november_2025", "name": "HMMT November 2025", "provenance": { "record_ids": [ "benchmark_hmmt_november_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_science", "id": "benchmark_hpct", "name": "HPCT", "provenance": { "record_ids": [ "benchmark_hpct" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_alignment", "id": "benchmark_human_sourced_jailbreaks", "name": "Human-Sourced Jailbreaks", "provenance": { "record_ids": [ "benchmark_human_sourced_jailbreaks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "HumanEval Python" ], "category_id": "category_coding", "id": "benchmark_humaneval", "name": "HumanEval", "provenance": { "record_ids": [ "benchmark_humaneval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_humaneval_infilling", "name": "HumanEval-Infilling", "provenance": { "record_ids": [ "benchmark_humaneval_infilling" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "HumanEval Multi", "HumanEval-Mul" ], "category_id": "category_coding", "id": "benchmark_humaneval_multilingual", "name": "HumanEval Multilingual", "provenance": { "record_ids": [ "benchmark_humaneval_multilingual" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "EvalPlus HumanEval" ], "category_id": "category_coding", "id": "benchmark_humaneval_plus", "name": "HumanEval Plus", "provenance": { "record_ids": [ "benchmark_humaneval_plus" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_hypersim", "name": "HyperSim", "provenance": { "record_ids": [ "benchmark_hypersim" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_icpc_wf_2025", "name": "ICPC WF 2025", "provenance": { "record_ids": [ "benchmark_icpc_wf_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_ifbench", "name": "IFBench", "provenance": { "record_ids": [ "benchmark_ifbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Instruction following", "Instruction Following Evaluation", "IF-Eval", "IFEval Benchmark" ], "category_id": "category_instruction_following", "id": "benchmark_ifeval", "name": "IFEval", "provenance": { "record_ids": [ "benchmark_ifeval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_iheval", "name": "IHEval", "provenance": { "record_ids": [ "benchmark_iheval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "IMO 2025" ], "category_id": "category_mathematics", "id": "benchmark_imo_2025", "name": "International Mathematical Olympiad 2025", "provenance": { "record_ids": [ "benchmark_imo_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematical_reasoning", "id": "benchmark_imo_2026", "name": "IMO 2026", "provenance": { "record_ids": [ "benchmark_imo_2026" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_imoanswerbench", "name": "IMOAnswerBench", "provenance": { "record_ids": [ "benchmark_imoanswerbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Claude Code Impossible Tasks" ], "category_id": "category_reward_hacking", "id": "benchmark_impossible_tasks", "name": "Impossible Tasks", "provenance": { "record_ids": [ "benchmark_impossible_tasks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_impossiblebench", "name": "ImpossibleBench", "provenance": { "record_ids": [ "benchmark_impossiblebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_incalmo_cyber_ranges", "name": "Incalmo Cyber Ranges", "provenance": { "record_ids": [ "benchmark_incalmo_cyber_ranges" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_include", "name": "INCLUDE", "provenance": { "record_ids": [ "benchmark_include" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_indommlu", "name": "IndoMMLU", "provenance": { "record_ids": [ "benchmark_indommlu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_infinitebench_en_mc", "name": "InfiniteBench En.MC", "provenance": { "record_ids": [ "benchmark_infinitebench_en_mc" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_infinitebench_en_qa", "name": "InfiniteBench En.QA", "provenance": { "record_ids": [ "benchmark_infinitebench_en_qa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "InfographicsQA", "InfoQA" ], "category_id": "category_image_understanding", "id": "benchmark_infographicsvqa", "name": "InfographicsVQA", "provenance": { "record_ids": [ "benchmark_infographicsvqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "InfoVQA" ], "category_id": "category_multimodal_understanding", "id": "benchmark_infographicvqa", "name": "InfographicVQA", "provenance": { "record_ids": [ "benchmark_infographicvqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_infovqa_test", "name": "InfoVQA Test", "provenance": { "record_ids": [ "benchmark_infovqa_test" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_instruction_hierarchy_conflicts", "name": "Instruction Hierarchy Conflicts", "provenance": { "record_ids": [ "benchmark_instruction_hierarchy_conflicts" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "AIRD" ], "category_id": "category_internal_ai_research_autonomy", "id": "benchmark_internal_ai_research_evaluation_suite", "name": "Internal AI Research Evaluation Suite", "provenance": { "record_ids": [ "benchmark_internal_ai_research_evaluation_suite" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "AI R&D Suite 1" ], "category_id": "category_ai_research_and_development", "id": "benchmark_internal_ai_research_evaluation_suite_1", "name": "Internal AI Research Evaluation Suite 1", "provenance": { "record_ids": [ "benchmark_internal_ai_research_evaluation_suite_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "AI R&D Suite 2" ], "category_id": "category_ai_research_and_development", "id": "benchmark_internal_ai_research_evaluation_suite_2", "name": "Internal AI Research Evaluation Suite 2", "provenance": { "record_ids": [ "benchmark_internal_ai_research_evaluation_suite_2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "computer use", "general tool use", "Model Context Protocol (MCP)" ], "category_id": "category_prompt_injection", "id": "benchmark_internal_prompt_injection_evaluations", "name": "Internal Prompt Injection Evaluations", "provenance": { "record_ids": [ "benchmark_internal_prompt_injection_evaluations" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_internal_sycophancy", "name": "Internal Sycophancy", "provenance": { "record_ids": [ "benchmark_internal_sycophancy" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "IChO 2025" ], "category_id": "category_mathematics", "id": "benchmark_international_chemistry_olympiad_2025", "name": "International Chemistry Olympiad 2025", "provenance": { "record_ids": [ "benchmark_international_chemistry_olympiad_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_international_physics_olympiad_2025", "name": "International Physics Olympiad 2025", "provenance": { "record_ids": [ "benchmark_international_physics_olympiad_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematical_reasoning", "id": "benchmark_international_physics_olympiad_2025_theory", "name": "International Physics Olympiad 2025 Theory", "provenance": { "record_ids": [ "benchmark_international_physics_olympiad_2025_theory" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_ioi_2025", "name": "IOI 2025", "provenance": { "record_ids": [ "benchmark_ioi_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_alignment", "id": "benchmark_jailbreak_augmented_examples", "name": "Jailbreak Augmented Examples", "provenance": { "record_ids": [ "benchmark_jailbreak_augmented_examples" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_jmmlu", "name": "JMMLU", "provenance": { "record_ids": [ "benchmark_jmmlu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_tasks", "id": "benchmark_jobbench", "name": "JobBench", "provenance": { "record_ids": [ "benchmark_jobbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_kernel_bench_l3", "name": "Kernel Bench L3", "provenance": { "record_ids": [ "benchmark_kernel_bench_l3" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_ai_research_and_development", "id": "benchmark_kernel_optimization", "name": "Kernel Optimization", "provenance": { "record_ids": [ "benchmark_kernel_optimization" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_kernelgen_1p", "name": "KernelGen 1P", "provenance": { "record_ids": [ "benchmark_kernelgen_1p" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_ai_research_and_development", "id": "benchmark_kernels_task", "name": "Kernels Task", "provenance": { "record_ids": [ "benchmark_kernels_task" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_kespeech", "name": "Kespeech", "provenance": { "record_ids": [ "benchmark_kespeech" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_kmmlu", "name": "KMMLU", "provenance": { "record_ids": [ "benchmark_kmmlu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_korbench", "name": "KoRBench", "provenance": { "record_ids": [ "benchmark_korbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_l_eval", "name": "L-Eval", "provenance": { "record_ids": [ "benchmark_l_eval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_biological_knowledge", "id": "benchmark_lab_bench", "name": "LAB-Bench", "provenance": { "record_ids": [ "benchmark_lab_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "FigQA is identified as part of Language Agent Biology Benchmark", "Identified as part of Language Agent Biology Benchmark", "Language Agent Biology Benchmark FigQA" ], "category_id": "category_biology_multimodal", "id": "benchmark_lab_bench_figqa", "name": "LAB-Bench FigQA", "provenance": { "record_ids": [ "benchmark_lab_bench_figqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LAB-Bench ProtocolQA" ], "category_id": "category_biological_knowledge", "id": "benchmark_lab_bench_protocolqa", "name": "ProtocolQA", "provenance": { "record_ids": [ "benchmark_lab_bench_protocolqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LAB-Bench SeqQA" ], "category_id": "category_biological_knowledge", "id": "benchmark_lab_bench_seqqa", "name": "SeqQA", "provenance": { "record_ids": [ "benchmark_lab_bench_seqqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LabBench evaluations", "LabBench Subset" ], "category_id": "category_biological_laboratory_skills", "id": "benchmark_labbench", "name": "LabBench", "provenance": { "record_ids": [ "benchmark_labbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_labbench2", "name": "LABBench2", "provenance": { "record_ids": [ "benchmark_labbench2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "cloning workflows" ], "category_id": "category_biological_cloning_workflows", "id": "benchmark_labbench_cloningscenarios", "name": "LabBench CloningScenarios", "provenance": { "record_ids": [ "benchmark_labbench_cloningscenarios" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "figure interpretation" ], "category_id": "category_biological_figure_interpretation", "id": "benchmark_labbench_figqa", "name": "LabBench FigQA", "provenance": { "record_ids": [ "benchmark_labbench_figqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "protocol understanding" ], "category_id": "category_biological_protocol_understanding", "id": "benchmark_labbench_protocolqa", "name": "LabBench ProtocolQA", "provenance": { "record_ids": [ "benchmark_labbench_protocolqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "DNA/protein sequence manipulation" ], "category_id": "category_biological_sequence_reasoning", "id": "benchmark_labbench_seqqa", "name": "LabBench SeqQA", "provenance": { "record_ids": [ "benchmark_labbench_seqqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_labworkbench", "name": "LabWorkBench", "provenance": { "record_ids": [ "benchmark_labworkbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "SpatialBench" ], "category_id": "category_life_sciences", "id": "benchmark_latchbio_bioinformatics", "name": "LatchBio Bioinformatics", "provenance": { "record_ids": [ "benchmark_latchbio_bioinformatics" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LSAT" ], "category_id": "category_standardized_legal_reasoning", "id": "benchmark_law_school_admission_test", "name": "Law School Admission Test", "provenance": { "record_ids": [ "benchmark_law_school_admission_test" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LCFO twenty percent condition" ], "category_id": "category_long_context", "id": "benchmark_lcfo_long_summary_condition", "name": "LCFO long-summary condition", "provenance": { "record_ids": [ "benchmark_lcfo_long_summary_condition" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LCFO ten percent condition" ], "category_id": "category_long_context", "id": "benchmark_lcfo_medium_summary_condition", "name": "LCFO medium-summary condition", "provenance": { "record_ids": [ "benchmark_lcfo_medium_summary_condition" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LCFO five percent condition" ], "category_id": "category_long_context", "id": "benchmark_lcfo_short_summary_condition", "name": "LCFO short-summary condition", "provenance": { "record_ids": [ "benchmark_lcfo_short_summary_condition" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LAB" ], "category_id": "category_professional_tasks", "id": "benchmark_legal_agent_benchmark", "name": "Legal Agent Benchmark", "provenance": { "record_ids": [ "benchmark_legal_agent_benchmark" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_librispeech", "name": "LibriSpeech", "provenance": { "record_ids": [ "benchmark_librispeech" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_lifescibench", "name": "LifeSciBench", "provenance": { "record_ids": [ "benchmark_lifescibench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_lingoqa", "name": "LingoQA", "provenance": { "record_ids": [ "benchmark_lingoqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safeguard_evasion", "id": "benchmark_linuxarena", "name": "LinuxArena", "provenance": { "record_ids": [ "benchmark_linuxarena" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_livebench", "name": "LiveBench", "provenance": { "record_ids": [ "benchmark_livebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_livebench_coding", "name": "LiveBench Coding", "provenance": { "record_ids": [ "benchmark_livebench_coding" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_livebench_july_2025", "name": "LiveBench July 2025", "provenance": { "record_ids": [ "benchmark_livebench_july_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LiveCodeBench date-window variants" ], "category_id": "category_coding", "id": "benchmark_livecodebench", "name": "LiveCodeBench", "provenance": { "record_ids": [ "benchmark_livecodebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LiveCodeBench-Base" ], "category_id": "category_coding", "id": "benchmark_livecodebench_base", "name": "LiveCodeBench Base", "provenance": { "record_ids": [ "benchmark_livecodebench_base" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_livecodebench_pro", "name": "LiveCodeBench Pro", "provenance": { "record_ids": [ "benchmark_livecodebench_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_livecodebench_v5", "name": "LiveCodeBench v5", "provenance": { "record_ids": [ "benchmark_livecodebench_v5" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_livecodebench_v6", "name": "LiveCodeBench v6", "provenance": { "record_ids": [ "benchmark_livecodebench_v6" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_ai_research_and_development", "id": "benchmark_llm_training", "name": "LLM Training", "provenance": { "record_ids": [ "benchmark_llm_training" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_ai_research_and_development", "id": "benchmark_llm_training_optimization", "name": "LLM Training Optimization", "provenance": { "record_ids": [ "benchmark_llm_training_optimization" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LMSYS leaderboard", "Chatbot Arena", "LMArena" ], "category_id": "category_general_reasoning", "id": "benchmark_lmarena", "name": "LMSYS Chatbot Arena", "provenance": { "record_ids": [ "benchmark_lmarena" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_logicvista", "name": "LogicVista", "provenance": { "record_ids": [ "benchmark_logicvista" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_biological_risk", "id": "benchmark_long_form_virology_task_1", "name": "Long-Form Virology Task 1", "provenance": { "record_ids": [ "benchmark_long_form_virology_task_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_biological_risk", "id": "benchmark_long_form_virology_task_2", "name": "Long-Form Virology Task 2", "provenance": { "record_ids": [ "benchmark_long_form_virology_task_2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_biological_risk_agentic_evaluation", "id": "benchmark_long_form_virology_tasks", "name": "Long-form Virology Tasks", "provenance": { "record_ids": [ "benchmark_long_form_virology_tasks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_longbench_chat", "name": "LongBench-Chat", "provenance": { "record_ids": [ "benchmark_longbench_chat" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "LongBench-V2" ], "category_id": "category_long_context", "id": "benchmark_longbench_v2", "name": "LongBench v2", "provenance": { "record_ids": [ "benchmark_longbench_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_factuality", "id": "benchmark_longfact", "name": "LongFact", "provenance": { "record_ids": [ "benchmark_longfact" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_longvideobench", "name": "LongVideoBench", "provenance": { "record_ids": [ "benchmark_longvideobench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_lv_eval", "name": "LV-Eval", "provenance": { "record_ids": [ "benchmark_lv_eval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_lvbench", "name": "LVBench", "provenance": { "record_ids": [ "benchmark_lvbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_lvbench_w_mem", "name": "LVBench (w/ Mem.)", "provenance": { "record_ids": [ "benchmark_lvbench_w_mem" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_m3exams", "name": "M3Exams", "provenance": { "record_ids": [ "benchmark_m3exams" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_m_mmlu", "name": "M-MMLU", "provenance": { "record_ids": [ "benchmark_m_mmlu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "MTOB" ], "category_id": "category_multilingual", "id": "benchmark_machine_translation_from_one_book", "name": "Machine Translation from One Book", "provenance": { "record_ids": [ "benchmark_machine_translation_from_one_book" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_madqa", "name": "MADQA", "provenance": { "record_ids": [ "benchmark_madqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_alignment", "id": "benchmark_makemepay", "name": "MakeMePay", "provenance": { "record_ids": [ "benchmark_makemepay" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_alignment", "id": "benchmark_makemesay", "name": "MakeMeSay", "provenance": { "record_ids": [ "benchmark_makemesay" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_many_shot_gsm8k", "name": "Many-shot GSM8K", "provenance": { "record_ids": [ "benchmark_many_shot_gsm8k" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "MASK", "Model Alignment between Statements and Knowledge (MASK)" ], "category_id": "category_honesty_and_hallucination", "id": "benchmark_mask", "name": "Model Alignment between Statements and Knowledge", "provenance": { "record_ids": [ "benchmark_mask" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "validation", "Visual question answering" ], "category_id": "category_multimodal_reasoning", "id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "name": "Massive Multi-discipline Multimodal Understanding and Reasoning", "provenance": { "record_ids": [ "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Massive Multitask Language Understanding" ], "category_id": "category_general_reasoning", "id": "benchmark_massive_multitask_language_understanding", "name": "MMLU", "provenance": { "record_ids": [ "benchmark_massive_multitask_language_understanding" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "MATH Dataset", "Mathematical problem solving", "Hendrycks MATH" ], "category_id": "category_mathematics", "id": "benchmark_math", "name": "MATH", "provenance": { "record_ids": [ "benchmark_math" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_math_500", "name": "MATH-500", "provenance": { "record_ids": [ "benchmark_math_500" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_matharena_apex", "name": "MathArena Apex", "provenance": { "record_ids": [ "benchmark_matharena_apex" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_mathverse", "name": "MathVerse", "provenance": { "record_ids": [ "benchmark_mathverse" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_mathvision", "name": "MathVision", "provenance": { "record_ids": [ "benchmark_mathvision" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "testmini" ], "category_id": "category_mathematics_and_science", "id": "benchmark_mathvista", "name": "MathVista", "provenance": { "record_ids": [ "benchmark_mathvista" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_mathvista_mini", "name": "MathVista Mini", "provenance": { "record_ids": [ "benchmark_mathvista_mini" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_maxife", "name": "MAXIFE", "provenance": { "record_ids": [ "benchmark_maxife" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_science", "id": "benchmark_mbct", "name": "MBCT", "provenance": { "record_ids": [ "benchmark_mbct" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Code generation", "MBPP" ], "category_id": "category_coding", "id": "benchmark_mbpp", "name": "Mostly Basic Python Problems", "provenance": { "record_ids": [ "benchmark_mbpp" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_mbpp_evalplus_base", "name": "MBPP EvalPlus Base", "provenance": { "record_ids": [ "benchmark_mbpp_evalplus_base" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_mbpp_plus", "name": "MBPP Plus", "provenance": { "record_ids": [ "benchmark_mbpp_plus" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_mceval", "name": "McEval", "provenance": { "record_ids": [ "benchmark_mceval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "MCP Atlas", "Scale MCP Atlas", "MCP-Atlas", "The launch footnote prints MCP Atlas", "The section body prints MCP-Atlas" ], "category_id": "category_tool_use", "id": "benchmark_mcp_atlas", "name": "Scale MCP-Atlas", "provenance": { "record_ids": [ "benchmark_mcp_atlas" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_tool_use", "id": "benchmark_mcp_mark", "name": "MCP-Mark", "provenance": { "record_ids": [ "benchmark_mcp_mark" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "MCP evaluation", "Model Context Protocol (MCP) evaluation" ], "category_id": "category_prompt_injection", "id": "benchmark_mcp_prompt_injection_evaluation", "name": "MCP Prompt Injection Evaluation", "provenance": { "record_ids": [ "benchmark_mcp_prompt_injection_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_tool_use", "id": "benchmark_mcp_universe", "name": "MCP-Universe", "provenance": { "record_ids": [ "benchmark_mcp_universe" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_tool_use", "id": "benchmark_mcpatlas", "name": "MCPAtlas", "provenance": { "record_ids": [ "benchmark_mcpatlas" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_tool_use", "id": "benchmark_mcpatlas_public", "name": "MCPAtlas Public", "provenance": { "record_ids": [ "benchmark_mcpatlas_public" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_mdeval", "name": "MdEval", "provenance": { "record_ids": [ "benchmark_mdeval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "The section first prints MedCalc-Bench and identifies use of the latest verified dataset" ], "category_id": "category_medical_reasoning", "id": "benchmark_medcalc_bench_verified", "name": "MedCalc-Bench Verified", "provenance": { "record_ids": [ "benchmark_medcalc_bench_verified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_medchembench", "name": "MedChemBench", "provenance": { "record_ids": [ "benchmark_medchembench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_medical_question_answering", "id": "benchmark_medmcqa", "name": "MedMCQA", "provenance": { "record_ids": [ "benchmark_medmcqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_healthcare", "id": "benchmark_medqa", "name": "MedQA", "provenance": { "record_ids": [ "benchmark_medqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_medxpertqa_mm", "name": "MedXpertQA-MM", "provenance": { "record_ids": [ "benchmark_medxpertqa_mm" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_science", "id": "benchmark_medxpertqa_multimodal", "name": "MedXpertQA Multimodal", "provenance": { "record_ids": [ "benchmark_medxpertqa_multimodal" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_science", "id": "benchmark_medxpertqa_text", "name": "MedXpertQA Text", "provenance": { "record_ids": [ "benchmark_medxpertqa_text" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_megabench", "name": "MegaBench", "provenance": { "record_ids": [ "benchmark_megabench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_meta_attack_automation_framework", "name": "Meta Attack Automation Framework", "provenance": { "record_ids": [ "benchmark_meta_attack_automation_framework" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_meta_spear_phishing_benchmark", "name": "Meta Spear Phishing Benchmark", "provenance": { "record_ids": [ "benchmark_meta_spear_phishing_benchmark" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "AIRD Data Deduplication" ], "category_id": "category_autonomy_and_software_engineering", "id": "benchmark_metr_data_deduplication", "name": "METR Data Deduplication", "provenance": { "record_ids": [ "benchmark_metr_data_deduplication" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Multilingual Grade School Math", "Multilingual Math", "Multilingual Math MGSM", "translated version of the math benchmark GSM8K" ], "category_id": "category_multilingual", "id": "benchmark_mgsm", "name": "MGSM", "provenance": { "record_ids": [ "benchmark_mgsm" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Expanded as Multi-task Indic Language Understanding Benchmark" ], "category_id": "category_multilingual_knowledge", "id": "benchmark_milu", "name": "MILU", "provenance": { "record_ids": [ "benchmark_milu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_minerva_math", "name": "Minerva Math", "provenance": { "record_ids": [ "benchmark_minerva_math" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safeguard_evasion", "id": "benchmark_minimal_linuxbench", "name": "Minimal-LinuxBench", "provenance": { "record_ids": [ "benchmark_minimal_linuxbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_minimax_multilingual", "name": "MiniMax Multilingual", "provenance": { "record_ids": [ "benchmark_minimax_multilingual" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_mir_1k", "name": "MIR-1K", "provenance": { "record_ids": [ "benchmark_mir_1k" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "MLE Bench" ], "category_id": "category_general_reasoning", "id": "benchmark_mle_bench", "name": "MLE-Bench", "provenance": { "record_ids": [ "benchmark_mle_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_mls_bench_lite", "name": "MLS-Bench-Lite", "provenance": { "record_ids": [ "benchmark_mls_bench_lite" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_mlvu", "name": "MLVU", "provenance": { "record_ids": [ "benchmark_mlvu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_human_preference", "id": "benchmark_mm_mt_bench", "name": "MM-MT-Bench", "provenance": { "record_ids": [ "benchmark_mm_mt_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_mmar", "name": "MMAR", "provenance": { "record_ids": [ "benchmark_mmar" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_mmau", "name": "MMAU", "provenance": { "record_ids": [ "benchmark_mmau" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_mmbc", "name": "MMBC", "provenance": { "record_ids": [ "benchmark_mmbc" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_mmbench", "name": "MMBench", "provenance": { "record_ids": [ "benchmark_mmbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_mmbench_1_1", "name": "MMBench 1.1", "provenance": { "record_ids": [ "benchmark_mmbench_1_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_mmbench_en_1_1_dev", "name": "MMBench EN 1.1 dev", "provenance": { "record_ids": [ "benchmark_mmbench_en_1_1_dev" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_mmbench_video", "name": "MMBench-Video", "provenance": { "record_ids": [ "benchmark_mmbench_video" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_mme", "name": "MME", "provenance": { "record_ids": [ "benchmark_mme" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_mme_videoocr", "name": "MME-VideoOCR", "provenance": { "record_ids": [ "benchmark_mme_videoocr" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_mmlongbench_doc", "name": "MMLongBench-Doc", "provenance": { "record_ids": [ "benchmark_mmlongbench_doc" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_mmlu_multi", "name": "MMLU-multi", "provenance": { "record_ids": [ "benchmark_mmlu_multi" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_mmlu_multilingual", "name": "MMLU Multilingual", "provenance": { "record_ids": [ "benchmark_mmlu_multilingual" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "MMLU Pro" ], "category_id": "category_general_reasoning_and_knowledge", "id": "benchmark_mmlu_pro", "name": "MMLU-Pro", "provenance": { "record_ids": [ "benchmark_mmlu_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_mmlu_prox", "name": "MMLU-ProX", "provenance": { "record_ids": [ "benchmark_mmlu_prox" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_reasoning", "id": "benchmark_mmlu_redux", "name": "MMLU-Redux", "provenance": { "record_ids": [ "benchmark_mmlu_redux" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_mmlu_stem", "name": "MMLU-STEM", "provenance": { "record_ids": [ "benchmark_mmlu_stem" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Multilingual MMLU", "Expanded as Multilingual Massive Multitask Language Understanding", "Multilingual MMLU evaluation", "Reasoning" ], "category_id": "category_multilingual", "id": "benchmark_mmmlu", "name": "MMMLU", "provenance": { "record_ids": [ "benchmark_mmmlu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_reasoning", "id": "benchmark_mmmlu_non_english", "name": "MMMLU-non-English", "provenance": { "record_ids": [ "benchmark_mmmlu_non_english" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Massive Multi-discipline Multimodal Understanding", "Expanded as Massive Multi-discipline Multimodal Understanding", "The summary table identifies the validation split" ], "category_id": "category_multimodal_understanding", "id": "benchmark_mmmu", "name": "MMMU", "provenance": { "record_ids": [ "benchmark_mmmu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "MMMUPro", "Described as an improved variant of MMMU", "MMMU-Pro" ], "category_id": "category_multimodal_understanding", "id": "benchmark_mmmu_pro", "name": "MMMU Pro", "provenance": { "record_ids": [ "benchmark_mmmu_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_mmmu_pro_standard", "name": "MMMU-Pro Standard", "provenance": { "record_ids": [ "benchmark_mmmu_pro_standard" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_mmmu_pro_vision", "name": "MMMU-Pro Vision", "provenance": { "record_ids": [ "benchmark_mmmu_pro_vision" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_mmsearchplus", "name": "MMSearchPlus", "provenance": { "record_ids": [ "benchmark_mmsearchplus" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_mmstar", "name": "MMStar", "provenance": { "record_ids": [ "benchmark_mmstar" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_mmsu", "name": "MMSU", "provenance": { "record_ids": [ "benchmark_mmsu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_human_preference", "id": "benchmark_mmt_bench", "name": "MMT-Bench", "provenance": { "record_ids": [ "benchmark_mmt_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_mmvet", "name": "MMVet", "provenance": { "record_ids": [ "benchmark_mmvet" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_mmvu", "name": "MMVU", "provenance": { "record_ids": [ "benchmark_mmvu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_mobileworld", "name": "MobileWorld", "provenance": { "record_ids": [ "benchmark_mobileworld" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_motionbench", "name": "MotionBench", "provenance": { "record_ids": [ "benchmark_motionbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_mrcr_1m", "name": "MRCR 1M", "provenance": { "record_ids": [ "benchmark_mrcr_1m" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_mrcr_v2", "name": "MRCR v2", "provenance": { "record_ids": [ "benchmark_mrcr_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "COCO Captions" ], "category_id": "category_image_understanding", "id": "benchmark_ms_coco_captions", "name": "MS-COCO Captions", "provenance": { "record_ids": [ "benchmark_ms_coco_captions" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_and_preference", "id": "benchmark_mt_bench", "name": "MT-Bench", "provenance": { "record_ids": [ "benchmark_mt_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_mtob_full_book", "name": "MTOB Full Book", "provenance": { "record_ids": [ "benchmark_mtob_full_book" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_mtob_half_book", "name": "MTOB Half Book", "provenance": { "record_ids": [ "benchmark_mtob_half_book" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_mtvqa", "name": "MTVQA", "provenance": { "record_ids": [ "benchmark_mtvqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_mtvqa_test", "name": "MTVQA-Test", "provenance": { "record_ids": [ "benchmark_mtvqa_test" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_muirbench", "name": "MUIRBENCH", "provenance": { "record_ids": [ "benchmark_muirbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_multi_exam", "name": "Multi-Exam", "provenance": { "record_ids": [ "benchmark_multi_exam" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Multi-factual evaluation" ], "category_id": "category_internal_factuality", "id": "benchmark_multi_factual", "name": "Multi-factual", "provenance": { "record_ids": [ "benchmark_multi_factual" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_multi_if", "name": "Multi-IF", "provenance": { "record_ids": [ "benchmark_multi_if" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_multi_mathematics", "name": "Multi-Mathematics", "provenance": { "record_ids": [ "benchmark_multi_mathematics" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_summarization", "id": "benchmark_multi_news", "name": "Multi-News", "provenance": { "record_ids": [ "benchmark_multi_news" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_multi_task_indic_language_understanding_benchmark", "name": "Multi-task Indic Language Understanding Benchmark", "provenance": { "record_ids": [ "benchmark_multi_task_indic_language_understanding_benchmark" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_multi_translation", "name": "Multi-Translation", "provenance": { "record_ids": [ "benchmark_multi_translation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_multi_understanding", "name": "Multi-Understanding", "provenance": { "record_ids": [ "benchmark_multi_understanding" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_multichallenge", "name": "MultiChallenge", "provenance": { "record_ids": [ "benchmark_multichallenge" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_multiif", "name": "MultiIF", "provenance": { "record_ids": [ "benchmark_multiif" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_reasoning", "id": "benchmark_multiloko", "name": "MultiLoKo", "provenance": { "record_ids": [ "benchmark_multiloko" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_multipl_e", "name": "MultiPL-E", "provenance": { "record_ids": [ "benchmark_multipl_e" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "MBE", "Multistate Bar Exam" ], "category_id": "category_standardized_legal_knowledge", "id": "benchmark_multistate_bar_examination", "name": "Multistate Bar Examination", "provenance": { "record_ids": [ "benchmark_multistate_bar_examination" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_mvbench", "name": "MVBench", "provenance": { "record_ids": [ "benchmark_mvbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "N2C" ], "category_id": "category_coding", "id": "benchmark_natural2code", "name": "Natural2Code", "provenance": { "record_ids": [ "benchmark_natural2code" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "NQ" ], "category_id": "category_question_answering", "id": "benchmark_natural_questions", "name": "Natural Questions", "provenance": { "record_ids": [ "benchmark_natural_questions" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_factuality", "id": "benchmark_naturalquestions", "name": "NaturalQuestions", "provenance": { "record_ids": [ "benchmark_naturalquestions" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Needle Haystack evaluation", "NIAH", "Needle-in-a-Haystack" ], "category_id": "category_long_context", "id": "benchmark_needle_in_a_haystack", "name": "Needle In A Haystack", "provenance": { "record_ids": [ "benchmark_needle_in_a_haystack" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_next_qa", "name": "NExT-QA", "provenance": { "record_ids": [ "benchmark_next_qa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_tool_use_and_agents", "id": "benchmark_nexus", "name": "Nexus", "provenance": { "record_ids": [ "benchmark_nexus" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Multi-needle" ], "category_id": "category_long_context", "id": "benchmark_nih_multi_needle", "name": "NIH Multi-needle", "provenance": { "record_ids": [ "benchmark_nih_multi_needle" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_nl2repo", "name": "NL2Repo", "provenance": { "record_ids": [ "benchmark_nl2repo" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_nocaps", "name": "NoCaps", "provenance": { "record_ids": [ "benchmark_nocaps" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_nova", "name": "NOVA", "provenance": { "record_ids": [ "benchmark_nova" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_ai_research_and_development", "id": "benchmark_novel_compiler", "name": "Novel Compiler", "provenance": { "record_ids": [ "benchmark_novel_compiler" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_nucleobench", "name": "Nucleobench", "provenance": { "record_ids": [ "benchmark_nucleobench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_nuscene", "name": "Nuscene", "provenance": { "record_ids": [ "benchmark_nuscene" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_objectron", "name": "Objectron", "provenance": { "record_ids": [ "benchmark_objectron" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_ocr_bench_v2_chinese", "name": "OCR-Bench-V2 Chinese", "provenance": { "record_ids": [ "benchmark_ocr_bench_v2_chinese" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_ocr_bench_v2_en_zh", "name": "OCR-Bench-V2 (EN/ZH)", "provenance": { "record_ids": [ "benchmark_ocr_bench_v2_en_zh" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_ocr_bench_v2_english", "name": "OCR-Bench-V2 English", "provenance": { "record_ids": [ "benchmark_ocr_bench_v2_english" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_ocrbench", "name": "OCRBench", "provenance": { "record_ids": [ "benchmark_ocrbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_ocrbenchv2", "name": "OCRBenchV2", "provenance": { "record_ids": [ "benchmark_ocrbenchv2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_odinw13", "name": "ODinW13", "provenance": { "record_ids": [ "benchmark_odinw13" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_document_reasoning", "id": "benchmark_officeqa", "name": "OfficeQA", "provenance": { "record_ids": [ "benchmark_officeqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "The launch page prints OfficeQA Pro in a Databricks quotation" ], "category_id": "category_professional_work", "id": "benchmark_officeqa_pro", "name": "OfficeQA Pro", "provenance": { "record_ids": [ "benchmark_officeqa_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "OKVQA" ], "category_id": "category_multimodal_understanding", "id": "benchmark_ok_vqa", "name": "OK-VQA", "provenance": { "record_ids": [ "benchmark_ok_vqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_okapi_mmlu", "name": "Okapi MMLU", "provenance": { "record_ids": [ "benchmark_okapi_mmlu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_olympiadbench", "name": "OlympiadBench", "provenance": { "record_ids": [ "benchmark_olympiadbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_omni_cloze", "name": "Omni-Cloze", "provenance": { "record_ids": [ "benchmark_omni_cloze" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_omnibench", "name": "OmniBench", "provenance": { "record_ids": [ "benchmark_omnibench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_omnidocbench", "name": "OmniDocBench", "provenance": { "record_ids": [ "benchmark_omnidocbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_omnidocbench_1_5", "name": "OmniDocBench 1.5", "provenance": { "record_ids": [ "benchmark_omnidocbench_1_5" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_omnigaia", "name": "OmniGAIA", "provenance": { "record_ids": [ "benchmark_omnigaia" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_onemillion_bench", "name": "$OneMillion-Bench", "provenance": { "record_ids": [ "benchmark_onemillion_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Online Mind2Web" ], "category_id": "category_computer_use", "id": "benchmark_online_mind2web", "name": "Online-Mind2Web", "provenance": { "record_ids": [ "benchmark_online_mind2web" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_open_rewrite", "name": "Open-Rewrite", "provenance": { "record_ids": [ "benchmark_open_rewrite" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_factuality", "id": "benchmark_openai_chatgpt_factuality", "name": "OpenAI ChatGPT Factuality", "provenance": { "record_ids": [ "benchmark_openai_chatgpt_factuality" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_openai_code_refactoring_tasks", "name": "OpenAI Code Refactoring Tasks", "provenance": { "record_ids": [ "benchmark_openai_code_refactoring_tasks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_openai_code_review_evaluation", "name": "OpenAI Code Review Evaluation", "provenance": { "record_ids": [ "benchmark_openai_code_review_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_openai_frontend_preference_evaluation", "name": "OpenAI Frontend Preference Evaluation", "provenance": { "record_ids": [ "benchmark_openai_frontend_preference_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_openai_internal_swe_tasks", "name": "OpenAI Internal SWE Tasks", "provenance": { "record_ids": [ "benchmark_openai_internal_swe_tasks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_work", "id": "benchmark_openai_investment_banking_spreadsheet_tasks", "name": "OpenAI Investment Banking Spreadsheet Tasks", "provenance": { "record_ids": [ "benchmark_openai_investment_banking_spreadsheet_tasks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_work", "id": "benchmark_openai_management_consulting_tasks", "name": "OpenAI Management Consulting Tasks", "provenance": { "record_ids": [ "benchmark_openai_management_consulting_tasks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "OpenAI MRCR" ], "category_id": "category_long_context", "id": "benchmark_openai_mrcr", "name": "OpenAI-MRCR", "provenance": { "record_ids": [ "benchmark_openai_mrcr" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Expanded as Multi-Round Co-Reference Resolution", "The launch page prints MRCR v2" ], "category_id": "category_long_context", "id": "benchmark_openai_mrcr_v2", "name": "OpenAI MRCR v2", "provenance": { "record_ids": [ "benchmark_openai_mrcr_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_reasoning", "id": "benchmark_openai_research_debugging_evaluation", "name": "OpenAI Research Debugging Evaluation", "provenance": { "record_ids": [ "benchmark_openai_research_debugging_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_reasoning", "id": "benchmark_openai_research_engineer_interview", "name": "OpenAI Research Engineer Interview", "provenance": { "record_ids": [ "benchmark_openai_research_engineer_interview" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "OBQA" ], "category_id": "category_reasoning_and_knowledge", "id": "benchmark_openbookqa", "name": "OpenBookQA", "provenance": { "record_ids": [ "benchmark_openbookqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_opencpop", "name": "Opencpop", "provenance": { "record_ids": [ "benchmark_opencpop" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_openeqa", "name": "OpenEQA", "provenance": { "record_ids": [ "benchmark_openeqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_software_reliability", "id": "benchmark_openrca", "name": "OpenRCA", "provenance": { "record_ids": [ "benchmark_openrca" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_interpretability", "id": "benchmark_opinion_elicitation", "name": "Opinion Elicitation", "provenance": { "record_ids": [ "benchmark_opinion_elicitation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_or_bench", "name": "OR-Bench", "provenance": { "record_ids": [ "benchmark_or_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_summarization", "id": "benchmark_orangesum", "name": "OrangeSum", "provenance": { "record_ids": [ "benchmark_orangesum" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_oss_fuzz", "name": "OSS-Fuzz", "provenance": { "record_ids": [ "benchmark_oss_fuzz" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Figures 8.9.4.A and 8.9.4.B print OSWorld-Verified for the evaluated setup", "OSWorld benchmark", "OSWorld-Verified framework" ], "category_id": "category_computer_use", "id": "benchmark_osworld", "name": "OSWorld", "provenance": { "record_ids": [ "benchmark_osworld" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_computer_use", "id": "benchmark_osworld_2_0", "name": "OSWorld 2.0", "provenance": { "record_ids": [ "benchmark_osworld_2_0" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_osworld_g", "name": "OSWorld-G", "provenance": { "record_ids": [ "benchmark_osworld_g" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "The launch section initially discusses the OSWorld benchmark family and states that this model uses OSWorld-Verified", "OSWorld Verified" ], "category_id": "category_computer_use", "id": "benchmark_osworld_verified", "name": "OSWorld-Verified", "provenance": { "record_ids": [ "benchmark_osworld_verified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_paperbench", "name": "PaperBench", "provenance": { "record_ids": [ "benchmark_paperbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_parametric_cad_bench", "name": "Parametric CAD Bench", "provenance": { "record_ids": [ "benchmark_parametric_cad_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_passkey_retrieval", "name": "Passkey Retrieval", "provenance": { "record_ids": [ "benchmark_passkey_retrieval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_paws", "name": "PAWS", "provenance": { "record_ids": [ "benchmark_paws" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_paws_x", "name": "PAWS-X", "provenance": { "record_ids": [ "benchmark_paws_x" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "PerceptionTest" ], "category_id": "category_video_understanding", "id": "benchmark_perception_test", "name": "Perception Test", "provenance": { "record_ids": [ "benchmark_perception_test" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_perception_test_mcvqa", "name": "Perception Test MCVQA", "provenance": { "record_ids": [ "benchmark_perception_test_mcvqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_perceptionbench", "name": "PerceptionBench", "provenance": { "record_ids": [ "benchmark_perceptionbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_factuality", "id": "benchmark_personqa", "name": "PersonQA", "provenance": { "record_ids": [ "benchmark_personqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_phonology_bench_g2p", "name": "Phonology Bench G2P", "provenance": { "record_ids": [ "benchmark_phonology_bench_g2p" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_phrase_and_password_protection", "name": "Phrase and Password Protection", "provenance": { "record_ids": [ "benchmark_phrase_and_password_protection" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_physicsfinals", "name": "PhysicsFinals", "provenance": { "record_ids": [ "benchmark_physicsfinals" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_phyx", "name": "PhyX", "provenance": { "record_ids": [ "benchmark_phyx" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_language_modeling", "id": "benchmark_pile_test", "name": "Pile-test", "provenance": { "record_ids": [ "benchmark_pile_test" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reasoning_and_knowledge", "id": "benchmark_piqa", "name": "PIQA", "provenance": { "record_ids": [ "benchmark_piqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_plawbench", "name": "PLawBench", "provenance": { "record_ids": [ "benchmark_plawbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "FGQA" ], "category_id": "category_video_understanding", "id": "benchmark_plm_videobench_fgqa", "name": "PLM-VideoBench FGQA", "provenance": { "record_ids": [ "benchmark_plm_videobench_fgqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "RCap" ], "category_id": "category_video_understanding", "id": "benchmark_plm_videobench_rcap", "name": "PLM-VideoBench RCap", "provenance": { "record_ids": [ "benchmark_plm_videobench_rcap" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "RDCap" ], "category_id": "category_video_understanding", "id": "benchmark_plm_videobench_rdcap", "name": "PLM-VideoBench RDCap", "provenance": { "record_ids": [ "benchmark_plm_videobench_rdcap" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "RTLoc" ], "category_id": "category_video_understanding", "id": "benchmark_plm_videobench_rtloc", "name": "PLM-VideoBench RTLoc", "provenance": { "record_ids": [ "benchmark_plm_videobench_rtloc" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "SGQA" ], "category_id": "category_video_understanding", "id": "benchmark_plm_videobench_sgqa", "name": "PLM-VideoBench SGQA", "provenance": { "record_ids": [ "benchmark_plm_videobench_sgqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_pmc_vqa", "name": "PMC-VQA", "provenance": { "record_ids": [ "benchmark_pmc_vqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_summarization", "id": "benchmark_pn_summary", "name": "PN-Summary", "provenance": { "record_ids": [ "benchmark_pn_summary" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_poly_guard_bench", "name": "Poly-Guard Bench", "provenance": { "record_ids": [ "benchmark_poly_guard_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_polymath", "name": "PolyMATH", "provenance": { "record_ids": [ "benchmark_polymath" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_pope", "name": "POPE", "provenance": { "record_ids": [ "benchmark_pope" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_prbench_finance", "name": "PRBench-Finance", "provenance": { "record_ids": [ "benchmark_prbench_finance" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_prbench_legal", "name": "PRBench-Legal", "provenance": { "record_ids": [ "benchmark_prbench_legal" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_presentbench", "name": "PresentBench", "provenance": { "record_ids": [ "benchmark_presentbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_interpretability_control", "id": "benchmark_pretraining_documents", "name": "Pretraining Documents", "provenance": { "record_ids": [ "benchmark_pretraining_documents" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_alignment", "id": "benchmark_production_jailbreaks", "name": "Production Jailbreaks", "provenance": { "record_ids": [ "benchmark_production_jailbreaks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_professional_ctf", "name": "Professional CTF", "provenance": { "record_ids": [ "benchmark_professional_ctf" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_professional_ctfs", "name": "Professional CTFs", "provenance": { "record_ids": [ "benchmark_professional_ctfs" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Multi-Agent ProgramBench" ], "category_id": "category_long_context_software_engineering", "id": "benchmark_programbench", "name": "ProgramBench", "provenance": { "record_ids": [ "benchmark_programbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Gutenberg" ], "category_id": "category_text_quality", "id": "benchmark_project_gutenberg", "name": "Project Gutenberg", "provenance": { "record_ids": [ "benchmark_project_gutenberg" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_prompted_evaluation_awareness", "name": "Prompted Evaluation Awareness", "provenance": { "record_ids": [ "benchmark_prompted_evaluation_awareness" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_life_sciences", "id": "benchmark_proteingym_hard", "name": "ProteinGym Hard", "provenance": { "record_ids": [ "benchmark_proteingym_hard" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_protocolqa_open_ended", "name": "ProtocolQA Open-Ended", "provenance": { "record_ids": [ "benchmark_protocolqa_open_ended" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Biomedical questions" ], "category_id": "category_biomedical_question_answering", "id": "benchmark_pubmedqa", "name": "PubMedQA", "provenance": { "record_ids": [ "benchmark_pubmedqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_qasper", "name": "Qasper", "provenance": { "record_ids": [ "benchmark_qasper" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reading_comprehension", "id": "benchmark_quac", "name": "QuAC", "provenance": { "record_ids": [ "benchmark_quac" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_ai_research_and_development", "id": "benchmark_quadruped_locomotion", "name": "Quadruped Locomotion", "provenance": { "record_ids": [ "benchmark_quadruped_locomotion" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_ai_research_and_development", "id": "benchmark_quadruped_reinforcement_learning", "name": "Quadruped Reinforcement Learning", "provenance": { "record_ids": [ "benchmark_quadruped_reinforcement_learning" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_qualcomminteractive", "name": "QualcommInteractive", "provenance": { "record_ids": [ "benchmark_qualcomminteractive" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "QuALITY: Question Answering with Long Input Texts, Yes!" ], "category_id": "category_long_context_reading_comprehension", "id": "benchmark_quality", "name": "QuALITY", "provenance": { "record_ids": [ "benchmark_quality" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics_and_science", "id": "benchmark_quantbench", "name": "QuantBench", "provenance": { "record_ids": [ "benchmark_quantbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_qwenblenderbench", "name": "QwenBlenderBench", "provenance": { "record_ids": [ "benchmark_qwenblenderbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_qwenchinesebench", "name": "QwenChineseBench", "provenance": { "record_ids": [ "benchmark_qwenchinesebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_qwenclaw", "name": "Qwenclaw", "provenance": { "record_ids": [ "benchmark_qwenclaw" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_qwenclawbench", "name": "QwenClawBench", "provenance": { "record_ids": [ "benchmark_qwenclawbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_qwenqoderbench", "name": "QwenQoderBench", "provenance": { "record_ids": [ "benchmark_qwenqoderbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_qwenreactbench", "name": "QwenReactBench", "provenance": { "record_ids": [ "benchmark_qwenreactbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_qwensvg", "name": "QwenSVG", "provenance": { "record_ids": [ "benchmark_qwensvg" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_qwensvgbench", "name": "QwenSVGBench", "provenance": { "record_ids": [ "benchmark_qwensvgbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_qwenswebench", "name": "QwenSWEBench", "provenance": { "record_ids": [ "benchmark_qwenswebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_qwenvision2code", "name": "QwenVision2Code", "provenance": { "record_ids": [ "benchmark_qwenvision2code" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_qwenvisualoffice", "name": "QwenVisualOffice", "provenance": { "record_ids": [ "benchmark_qwenvisualoffice" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_qwenwebbench", "name": "QwenWebBench", "provenance": { "record_ids": [ "benchmark_qwenwebbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_qwenwebdev", "name": "QwenWebDev", "provenance": { "record_ids": [ "benchmark_qwenwebdev" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_qwenworldbench", "name": "QwenWorldBench", "provenance": { "record_ids": [ "benchmark_qwenworldbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reading_comprehension", "id": "benchmark_race", "name": "RACE", "provenance": { "record_ids": [ "benchmark_race" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "RACE-H", "Reading comprehension", "RACE-High" ], "category_id": "category_reading_comprehension", "id": "benchmark_race_high", "name": "RACE High", "provenance": { "record_ids": [ "benchmark_race_high" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "RACE-Middle" ], "category_id": "category_reading_comprehension", "id": "benchmark_race_middle", "name": "RACE Middle", "provenance": { "record_ids": [ "benchmark_race_middle" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_software_engineering", "id": "benchmark_rakuten_swe_bench", "name": "Rakuten-SWE-Bench", "provenance": { "record_ids": [ "benchmark_rakuten_swe_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_interpretability_control", "id": "benchmark_random_chat_questions", "name": "Random Chat Questions", "provenance": { "record_ids": [ "benchmark_random_chat_questions" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "RE-Bench" ], "category_id": "category_research_engineering", "id": "benchmark_re_bench_subset", "name": "RE-Bench subset", "provenance": { "record_ids": [ "benchmark_re_bench_subset" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Table 2.12.1.A identifies it as an internal evaluation", "Table 2.14.2 identifies it as an internal evaluation" ], "category_id": "category_finance", "id": "benchmark_real_world_finance", "name": "Real-World Finance", "provenance": { "record_ids": [ "benchmark_real_world_finance" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_tasks", "id": "benchmark_real_world_finance_v1", "name": "Real-World Finance v1", "provenance": { "record_ids": [ "benchmark_real_world_finance_v1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_professional_tasks", "id": "benchmark_real_world_finance_v2", "name": "Real-World Finance v2", "provenance": { "record_ids": [ "benchmark_real_world_finance_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_responsibility", "id": "benchmark_realtoxicity", "name": "RealToxicity", "provenance": { "record_ids": [ "benchmark_realtoxicity" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_realworldqa", "name": "RealWorldQA", "provenance": { "record_ids": [ "benchmark_realworldqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "100Q", "100Q Hard factual evaluation", "100Q-Hard" ], "category_id": "category_factuality", "id": "benchmark_record_100q_hard", "name": "100Q Hard", "provenance": { "record_ids": [ "benchmark_record_100q_hard" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "One-Hour VideoQA" ], "category_id": "category_long_context_understanding", "id": "benchmark_record_1h_videoqa", "name": "1H-VideoQA", "provenance": { "record_ids": [ "benchmark_record_1h_videoqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_tool_use", "id": "benchmark_record_2_bench", "name": "τ2-bench", "provenance": { "record_ids": [ "benchmark_record_2_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Airline (corrected)", "Described as correcting task-setup and grading issues in Airline" ], "category_id": "category_tool_use", "id": "benchmark_record_2_bench_airline_corrected", "name": "τ2-bench Airline Corrected", "provenance": { "record_ids": [ "benchmark_record_2_bench_airline_corrected" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Airline (original)", "The launch page refers to the airline scenario as τ2-bench" ], "category_id": "category_tool_use", "id": "benchmark_record_2_bench_airline_original", "name": "τ2-bench Airline Original", "provenance": { "record_ids": [ "benchmark_record_2_bench_airline_original" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "τ2-bench Retail", "τ²-bench (Retail)", "τ²-bench Retail" ], "category_id": "category_tool_use", "id": "benchmark_record_2_bench_retail", "name": "Tau2-bench Retail", "provenance": { "record_ids": [ "benchmark_record_2_bench_retail" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "τ2-bench Telecom", "τ²-bench (Telecom)" ], "category_id": "category_tool_use", "id": "benchmark_record_2_bench_telecom", "name": "Tau2-bench Telecom", "provenance": { "record_ids": [ "benchmark_record_2_bench_telecom" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_recreationbench", "name": "RecreationBench", "provenance": { "record_ids": [ "benchmark_recreationbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_redcode", "name": "RedCode", "provenance": { "record_ids": [ "benchmark_redcode" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_refadv_s", "name": "RefAdv-S", "provenance": { "record_ids": [ "benchmark_refadv_s" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_refcoco", "name": "RefCOCO", "provenance": { "record_ids": [ "benchmark_refcoco" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_refcoco_plus", "name": "RefCOCO Plus", "provenance": { "record_ids": [ "benchmark_refcoco_plus" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_refcocog", "name": "RefCOCOg", "provenance": { "record_ids": [ "benchmark_refcocog" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_refspatialbench", "name": "RefSpatialBench", "provenance": { "record_ids": [ "benchmark_refspatialbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_repoeval", "name": "RepoEval", "provenance": { "record_ids": [ "benchmark_repoeval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_interpretability", "id": "benchmark_research_sabotage", "name": "Research Sabotage", "provenance": { "record_ids": [ "benchmark_research_sabotage" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "research sandbagging evaluation" ], "category_id": "category_sabotage_capability", "id": "benchmark_research_sandbagging", "name": "Research Sandbagging", "provenance": { "record_ids": [ "benchmark_research_sandbagging" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reward_hacking", "id": "benchmark_reward_hack_prone_coding_tasks_v2", "name": "Reward-Hack-Prone Coding Tasks v2", "provenance": { "record_ids": [ "benchmark_reward_hack_prone_coding_tasks_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematical_reasoning", "id": "benchmark_riemannbench", "name": "RiemannBench", "provenance": { "record_ids": [ "benchmark_riemannbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_robospatialhome", "name": "RoboSpatialHome", "provenance": { "record_ids": [ "benchmark_robospatialhome" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "ROC-stories" ], "category_id": "category_text_quality", "id": "benchmark_rocstories", "name": "ROCStories", "provenance": { "record_ids": [ "benchmark_rocstories" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_rul_muchomusic", "name": "RUL-MuchoMusic", "provenance": { "record_ids": [ "benchmark_rul_muchomusic" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_ruler", "name": "RULER", "provenance": { "record_ids": [ "benchmark_ruler" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_safim", "name": "SAFIM", "provenance": { "record_ids": [ "benchmark_safim" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_summarization", "id": "benchmark_samsum", "name": "SAMSum", "provenance": { "record_ids": [ "benchmark_samsum" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_sat_math", "name": "SAT Math", "provenance": { "record_ids": [ "benchmark_sat_math" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_knowledge", "id": "benchmark_sat_reading", "name": "SAT Reading", "provenance": { "record_ids": [ "benchmark_sat_reading" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_save_bench", "name": "SAVE-Bench", "provenance": { "record_ids": [ "benchmark_save_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_scicode", "name": "SciCode", "provenance": { "record_ids": [ "benchmark_scicode" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_screenspot", "name": "ScreenSpot", "provenance": { "record_ids": [ "benchmark_screenspot" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "ScreenSpot-Pro" ], "category_id": "category_multimodal_understanding", "id": "benchmark_screenspot_pro", "name": "ScreenSpot Pro", "provenance": { "record_ids": [ "benchmark_screenspot_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_seal_0", "name": "Seal-0", "provenance": { "record_ids": [ "benchmark_seal_0" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_cybersecurity", "id": "benchmark_sec_bench_pro", "name": "SEC-Bench Pro", "provenance": { "record_ids": [ "benchmark_sec_bench_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_seccodebench", "name": "SecCodeBench", "provenance": { "record_ids": [ "benchmark_seccodebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Multimodal Virology", "multiple select variant" ], "category_id": "category_multimodal_virology_knowledge", "id": "benchmark_secure_bio_lab_assistance_benchmark_multimodal", "name": "Secure Bio Lab Assistance Benchmark - Multimodal", "provenance": { "record_ids": [ "benchmark_secure_bio_lab_assistance_benchmark_multimodal" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_secure_code_generation", "name": "Secure Code Generation", "provenance": { "record_ids": [ "benchmark_secure_code_generation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "DNA Synthesis Screening Evasion evaluation", "Synthesis Screening Evasion evaluation" ], "category_id": "category_biological_risk", "id": "benchmark_securebio_dna_synthesis_screening_evasion", "name": "SecureBio DNA Synthesis Screening Evasion", "provenance": { "record_ids": [ "benchmark_securebio_dna_synthesis_screening_evasion" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "SecureBio Virology Capability Test (VCT)" ], "category_id": "category_biological_risk", "id": "benchmark_securebio_virology_capability_test", "name": "SecureBio Virology Capability Test", "provenance": { "record_ids": [ "benchmark_securebio_virology_capability_test" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "SEED" ], "category_id": "category_image_understanding", "id": "benchmark_seed_bench", "name": "SEED-Bench", "provenance": { "record_ids": [ "benchmark_seed_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_seed_tts_eval", "name": "Seed-TTS-Eval", "provenance": { "record_ids": [ "benchmark_seed_tts_eval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_severe_cybermisuse_refusal", "name": "Severe Cybermisuse Refusal", "provenance": { "record_ids": [ "benchmark_severe_cybermisuse_refusal" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_sfiction", "name": "SFiction", "provenance": { "record_ids": [ "benchmark_sfiction" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "SHADE Arena" ], "category_id": "category_sabotage_capability", "id": "benchmark_shade_arena", "name": "SHADE-Arena", "provenance": { "record_ids": [ "benchmark_shade_arena" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_biological_risk", "id": "benchmark_short_horizon_computational_biology_tasks", "name": "Short-Horizon Computational Biology Tasks", "provenance": { "record_ids": [ "benchmark_short_horizon_computational_biology_tasks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_sifo", "name": "SIFO", "provenance": { "record_ids": [ "benchmark_sifo" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_sifo_multiturn", "name": "SIFO-multiturn", "provenance": { "record_ids": [ "benchmark_sifo_multiturn" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Simple-QA" ], "category_id": "category_factuality", "id": "benchmark_simpleqa", "name": "SimpleQA", "provenance": { "record_ids": [ "benchmark_simpleqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Simple-QA verified", "SimpleQA-Verified" ], "category_id": "category_factuality", "id": "benchmark_simpleqa_verified", "name": "SimpleQA Verified", "provenance": { "record_ids": [ "benchmark_simpleqa_verified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_simplevqa", "name": "SimpleVQA", "provenance": { "record_ids": [ "benchmark_simplevqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_skillsbench", "name": "SkillsBench", "provenance": { "record_ids": [ "benchmark_skillsbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_slake", "name": "SLAKE", "provenance": { "record_ids": [ "benchmark_slake" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_social_engineering_evaluation", "name": "Social Engineering Evaluation", "provenance": { "record_ids": [ "benchmark_social_engineering_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_social_engineering_refusal", "name": "Social Engineering Refusal", "provenance": { "record_ids": [ "benchmark_social_engineering_refusal" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_commonsense_reasoning", "id": "benchmark_social_iqa", "name": "Social IQa", "provenance": { "record_ids": [ "benchmark_social_iqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reasoning_and_knowledge", "id": "benchmark_socialiqa", "name": "SocialIQA", "provenance": { "record_ids": [ "benchmark_socialiqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_songformbench_chinese", "name": "SongFormBench Chinese", "provenance": { "record_ids": [ "benchmark_songformbench_chinese" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_songformbench_harmonixset", "name": "SongFormBench HarmonixSet", "provenance": { "record_ids": [ "benchmark_songformbench_harmonixset" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_spatialsense_vqa", "name": "SpatialSense VQA", "provenance": { "record_ids": [ "benchmark_spatialsense_vqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_speechrole", "name": "SpeechRole", "provenance": { "record_ids": [ "benchmark_speechrole" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_spider", "name": "Spider", "provenance": { "record_ids": [ "benchmark_spider" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_office_productivity", "id": "benchmark_spreadsheetbench", "name": "SpreadsheetBench", "provenance": { "record_ids": [ "benchmark_spreadsheetbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_spreadsheetbench_v1", "name": "SpreadSheetBench-v1", "provenance": { "record_ids": [ "benchmark_spreadsheetbench_v1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reading_comprehension", "id": "benchmark_squad_v2", "name": "SQuAD v2", "provenance": { "record_ids": [ "benchmark_squad_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_context", "id": "benchmark_squality", "name": "SQuALITY", "provenance": { "record_ids": [ "benchmark_squality" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_alignment", "id": "benchmark_standard_refusal_evaluation", "name": "Standard Refusal Evaluation", "provenance": { "record_ids": [ "benchmark_standard_refusal_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "STAR" ], "category_id": "category_video_understanding", "id": "benchmark_star_video_qa", "name": "STAR Video QA", "provenance": { "record_ids": [ "benchmark_star_video_qa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_alignment", "id": "benchmark_strongreject", "name": "StrongREJECT", "provenance": { "record_ids": [ "benchmark_strongreject" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_strongreject_v2", "name": "StrongREJECT v2", "provenance": { "record_ids": [ "benchmark_strongreject_v2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Subversion Strategy task" ], "category_id": "category_sabotage_capability", "id": "benchmark_subversion_strategy", "name": "Subversion Strategy", "provenance": { "record_ids": [ "benchmark_subversion_strategy" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_sunrgbd", "name": "SUNRGBD", "provenance": { "record_ids": [ "benchmark_sunrgbd" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_horizon_agents", "id": "benchmark_super_agent_benchmark", "name": "Super-Agent Benchmark", "provenance": { "record_ids": [ "benchmark_super_agent_benchmark" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reasoning_and_knowledge", "id": "benchmark_superglue", "name": "SuperGLUE", "provenance": { "record_ids": [ "benchmark_superglue" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_reasoning", "id": "benchmark_supergpqa", "name": "SuperGPQA", "provenance": { "record_ids": [ "benchmark_supergpqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_surds", "name": "SURDS", "provenance": { "record_ids": [ "benchmark_surds" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_swe_agentless", "name": "SWE-agentless", "provenance": { "record_ids": [ "benchmark_swe_agentless" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "The launch page prints SWE-bench Multilingual", "SWE Multilingual" ], "category_id": "category_coding", "id": "benchmark_swe_bench_multilingual", "name": "SWE-bench Multilingual", "provenance": { "record_ids": [ "benchmark_swe_bench_multilingual" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_software_engineering", "id": "benchmark_swe_bench_multimodal", "name": "SWE-bench Multimodal", "provenance": { "record_ids": [ "benchmark_swe_bench_multimodal" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "SWE-Bench Pro (public)", "Described as the Scale AI variant of SWE-bench" ], "category_id": "category_coding", "id": "benchmark_swe_bench_pro", "name": "SWE-Bench Pro", "provenance": { "record_ids": [ "benchmark_swe_bench_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "SWEbench Verified", "Software Engineering Bench Verified", "SWE-bench", "SWE-bench is expanded as Software Engineering Bench", "SWE Verified", "SWE-verified" ], "category_id": "category_coding", "id": "benchmark_swe_bench_verified", "name": "SWE-bench Verified", "provenance": { "record_ids": [ "benchmark_swe_bench_verified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_swe_bench_verified_hard", "name": "SWE-Bench Verified Hard", "provenance": { "record_ids": [ "benchmark_swe_bench_verified_hard" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "hard subset of SWE-bench Verified", "SWE-bench benchmark", "SWE-bench Verified (hard subset)" ], "category_id": "category_autonomy_and_software_engineering", "id": "benchmark_swe_bench_verified_hard_subset", "name": "SWE-bench Verified Hard Subset", "provenance": { "record_ids": [ "benchmark_swe_bench_verified_hard_subset" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_swe_lancer", "name": "SWE-Lancer", "provenance": { "record_ids": [ "benchmark_swe_lancer" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_swe_lancer_diamond", "name": "SWE-Lancer Diamond", "provenance": { "record_ids": [ "benchmark_swe_lancer_diamond" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_swe_lancer_ic_diamond", "name": "SWE-Lancer IC Diamond", "provenance": { "record_ids": [ "benchmark_swe_lancer_ic_diamond" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_swe_lancer_ic_swe", "name": "SWE-Lancer IC SWE", "provenance": { "record_ids": [ "benchmark_swe_lancer_ic_swe" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_swe_pi_agent", "name": "SWE-PI Agent", "provenance": { "record_ids": [ "benchmark_swe_pi_agent" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_swe_pi_synthetic", "name": "SWE-PI Synthetic", "provenance": { "record_ids": [ "benchmark_swe_pi_synthetic" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_swe_pro", "name": "SWE Pro", "provenance": { "record_ids": [ "benchmark_swe_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_swebench_pretrain", "name": "SWEBench-Pretrain", "provenance": { "record_ids": [ "benchmark_swebench_pretrain" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_interpretability", "id": "benchmark_sycophancy_eval", "name": "Sycophancy Eval", "provenance": { "record_ids": [ "benchmark_sycophancy_eval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Synthesis Screening Evasion Task" ], "category_id": "category_biological_risk", "id": "benchmark_synthesis_screening_evasion", "name": "Synthesis Screening Evasion", "provenance": { "record_ids": [ "benchmark_synthesis_screening_evasion" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_interpretability", "id": "benchmark_synthetic_honeypots", "name": "Synthetic Honeypots", "provenance": { "record_ids": [ "benchmark_synthetic_honeypots" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_tallyqa", "name": "TallyQA", "provenance": { "record_ids": [ "benchmark_tallyqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_tat_dqa", "name": "TAT-DQA", "provenance": { "record_ids": [ "benchmark_tat_dqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_tool_use", "id": "benchmark_tau2_bench", "name": "Tau2-Bench", "provenance": { "record_ids": [ "benchmark_tau2_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_tau3_bench", "name": "TAU3-Bench", "provenance": { "record_ids": [ "benchmark_tau3_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Taubench airline" ], "category_id": "category_tool_use", "id": "benchmark_tau_bench_airline", "name": "Tau-bench Airline", "provenance": { "record_ids": [ "benchmark_tau_bench_airline" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Taubench retail" ], "category_id": "category_tool_use", "id": "benchmark_tau_bench_retail", "name": "Tau-bench Retail", "provenance": { "record_ids": [ "benchmark_tau_bench_retail" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "τ-Bench Telecom" ], "category_id": "category_tool_use_and_agents", "id": "benchmark_tau_bench_telecom", "name": "Tau-Bench Telecom", "provenance": { "record_ids": [ "benchmark_tau_bench_telecom" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_tempcompass", "name": "TempCompass", "provenance": { "record_ids": [ "benchmark_tempcompass" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_temporalbench", "name": "TemporalBench", "provenance": { "record_ids": [ "benchmark_temporalbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "TerminalBench" ], "category_id": "category_coding", "id": "benchmark_terminal_bench", "name": "Terminal-Bench", "provenance": { "record_ids": [ "benchmark_terminal_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "A launch quotation prints Terminal Bench", "Launch quotations also print Terminal Bench and TBench", "The section heading prints Terminal-Bench while the method names Terminal-Bench 2.0", "Terminal Bench 2.0" ], "category_id": "category_coding", "id": "benchmark_terminal_bench_2_0", "name": "Terminal-Bench 2.0", "provenance": { "record_ids": [ "benchmark_terminal_bench_2_0" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Terminal Bench 2.1" ], "category_id": "category_coding", "id": "benchmark_terminal_bench_2_1", "name": "Terminal-Bench 2.1", "provenance": { "record_ids": [ "benchmark_terminal_bench_2_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_human_preference", "id": "benchmark_text_arena", "name": "Text Arena", "provenance": { "record_ids": [ "benchmark_text_arena" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Text-based RL" ], "category_id": "category_ai_research_and_development", "id": "benchmark_text_based_reinforcement_learning", "name": "Text-Based Reinforcement Learning", "provenance": { "record_ids": [ "benchmark_text_based_reinforcement_learning" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_ai_research_and_development", "id": "benchmark_text_based_reinforcement_learning_task", "name": "Text-Based Reinforcement Learning Task", "provenance": { "record_ids": [ "benchmark_text_based_reinforcement_learning_task" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Text VQA" ], "category_id": "category_multimodal_understanding", "id": "benchmark_textvqa", "name": "TextVQA", "provenance": { "record_ids": [ "benchmark_textvqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_summarization", "id": "benchmark_thaisum", "name": "ThaiSum", "provenance": { "record_ids": [ "benchmark_thaisum" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_theoremqa", "name": "TheoremQA", "provenance": { "record_ids": [ "benchmark_theoremqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_ai_research_and_development", "id": "benchmark_time_series_forecasting", "name": "Time Series Forecasting", "provenance": { "record_ids": [ "benchmark_time_series_forecasting" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_tir_bench", "name": "TIR-Bench", "provenance": { "record_ids": [ "benchmark_tir_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_summarization", "id": "benchmark_tldr9", "name": "TLDR9+", "provenance": { "record_ids": [ "benchmark_tldr9" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_toefl", "name": "TOEFL", "provenance": { "record_ids": [ "benchmark_toefl" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_tomato", "name": "TOMATO", "provenance": { "record_ids": [ "benchmark_tomato" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agentic_tool_use", "id": "benchmark_tool_decathlon", "name": "Tool-Decathlon", "provenance": { "record_ids": [ "benchmark_tool_decathlon" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Tool use evaluation" ], "category_id": "category_prompt_injection", "id": "benchmark_tool_use_prompt_injection_evaluation", "name": "Tool-Use Prompt Injection Evaluation", "provenance": { "record_ids": [ "benchmark_tool_use_prompt_injection_evaluation" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_tool_use", "id": "benchmark_toolathlon", "name": "Toolathlon", "provenance": { "record_ids": [ "benchmark_toolathlon" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Toolathlon-Verified" ], "category_id": "category_tool_use", "id": "benchmark_toolathlon_verified", "name": "Toolathlon Verified", "provenance": { "record_ids": [ "benchmark_toolathlon_verified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_toolcallformatifbench", "name": "ToolcallFormatIFBench", "provenance": { "record_ids": [ "benchmark_toolcallformatifbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_responsibility", "id": "benchmark_toxigen", "name": "ToxiGen", "provenance": { "record_ids": [ "benchmark_toxigen" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "TQA" ], "category_id": "category_question_answering", "id": "benchmark_triviaqa", "name": "TriviaQA", "provenance": { "record_ids": [ "benchmark_triviaqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_reading_comprehension", "id": "benchmark_triviaqa_wiki", "name": "TriviaQA-Wiki", "provenance": { "record_ids": [ "benchmark_triviaqa_wiki" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_responsibility", "id": "benchmark_truthfulqa", "name": "TruthfulQA", "provenance": { "record_ids": [ "benchmark_truthfulqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_turkishmmlu", "name": "TurkishMMLU", "provenance": { "record_ids": [ "benchmark_turkishmmlu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_instruction_following", "id": "benchmark_tutor_jailbreaks", "name": "Tutor Jailbreaks", "provenance": { "record_ids": [ "benchmark_tutor_jailbreaks" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_tvbench", "name": "TVBench", "provenance": { "record_ids": [ "benchmark_tvbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_tydiqa", "name": "TyDiQA", "provenance": { "record_ids": [ "benchmark_tydiqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_unisvg", "name": "UniSVG", "provenance": { "record_ids": [ "benchmark_unisvg" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "USMLE" ], "category_id": "category_medical_knowledge", "id": "benchmark_united_states_medical_licensing_examination", "name": "United States Medical Licensing Examination", "provenance": { "record_ids": [ "benchmark_united_states_medical_licensing_examination" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_uro_bench_pro", "name": "URO-Bench-Pro", "provenance": { "record_ids": [ "benchmark_uro_bench_pro" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "2025 USAMO" ], "category_id": "category_mathematics", "id": "benchmark_usamo_2025", "name": "USAMO 2025", "provenance": { "record_ids": [ "benchmark_usamo_2025" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Grading follows MathArena methodology", "USA Mathematical Olympiad 2026", "USAMO is expanded as USA Mathematical Olympiad" ], "category_id": "category_mathematical_reasoning", "id": "benchmark_usamo_2026", "name": "USAMO 2026", "provenance": { "record_ids": [ "benchmark_usamo_2026" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_v", "name": "V*", "provenance": { "record_ids": [ "benchmark_v" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_v_bench", "name": "V* Bench", "provenance": { "record_ids": [ "benchmark_v_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Vals AI public leaderboard" ], "category_id": "category_finance_agent", "id": "benchmark_vals_ai_finance_agent", "name": "Vals AI Finance Agent", "provenance": { "record_ids": [ "benchmark_vals_ai_finance_agent" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_vatex", "name": "VATEX", "provenance": { "record_ids": [ "benchmark_vatex" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_vatex_zh", "name": "VATEX ZH", "provenance": { "record_ids": [ "benchmark_vatex_zh" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_science", "id": "benchmark_vct", "name": "VCT", "provenance": { "record_ids": [ "benchmark_vct" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_long_horizon_agents", "id": "benchmark_vending_bench", "name": "Vending-Bench", "provenance": { "record_ids": [ "benchmark_vending_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "The launch footnote prints Vending-Bench 2", "The launch page also prints Vending-Bench", "The section body identifies the benchmark as Vending-Bench 2", "VendingBench" ], "category_id": "category_long_horizon_agents", "id": "benchmark_vending_bench_2", "name": "Vending-Bench 2", "provenance": { "record_ids": [ "benchmark_vending_bench_2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "The launch page distinguishes this competitive Arena evaluation from Vending-Bench 2" ], "category_id": "category_long_horizon_agents", "id": "benchmark_vending_bench_arena", "name": "Vending-Bench Arena", "provenance": { "record_ids": [ "benchmark_vending_bench_arena" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_coding", "id": "benchmark_vibecodebench_v1_1", "name": "VibeCodeBench v1.1", "provenance": { "record_ids": [ "benchmark_vibecodebench_v1_1" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "end-to-end vibe-coding benchmark" ], "category_id": "category_software_engineering", "id": "benchmark_vibench", "name": "ViBench", "provenance": { "record_ids": [ "benchmark_vibench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_video_mme", "name": "Video-MME", "provenance": { "record_ids": [ "benchmark_video_mme" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Video-MMMU" ], "category_id": "category_multimodal_understanding", "id": "benchmark_video_mmmu", "name": "Video MMMU", "provenance": { "record_ids": [ "benchmark_video_mmmu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_videodr_w_search", "name": "VideoDR (w/ Search)", "provenance": { "record_ids": [ "benchmark_videodr_w_search" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_videohallucer", "name": "VideoHallucer", "provenance": { "record_ids": [ "benchmark_videohallucer" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_videomme", "name": "VideoMME", "provenance": { "record_ids": [ "benchmark_videomme" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_videomme_v2_w_sub", "name": "VideoMME v2 (w/ Sub.)", "provenance": { "record_ids": [ "benchmark_videomme_v2_w_sub" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_videomme_with_audio", "name": "VideoMME with audio", "provenance": { "record_ids": [ "benchmark_videomme_with_audio" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_videomme_with_subtitles", "name": "VideoMME with subtitles", "provenance": { "record_ids": [ "benchmark_videomme_with_subtitles" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_videomme_without_subtitles", "name": "VideoMME without subtitles", "provenance": { "record_ids": [ "benchmark_videomme_without_subtitles" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_videommmu", "name": "VideoMMMU", "provenance": { "record_ids": [ "benchmark_videommmu" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_visfactor", "name": "VisFactor", "provenance": { "record_ids": [ "benchmark_visfactor" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_vision2web", "name": "Vision2Web", "provenance": { "record_ids": [ "benchmark_vision2web" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_visulogic", "name": "VisuLogic", "provenance": { "record_ids": [ "benchmark_visulogic" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_vita_bench", "name": "VITA-Bench", "provenance": { "record_ids": [ "benchmark_vita_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_vizwiz", "name": "VizWiz", "provenance": { "record_ids": [ "benchmark_vizwiz" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_video_understanding", "id": "benchmark_vladbench", "name": "VLADBench", "provenance": { "record_ids": [ "benchmark_vladbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_vlmsarebiased", "name": "VLMsAreBiased", "provenance": { "record_ids": [ "benchmark_vlmsarebiased" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_vlmsareblind", "name": "VlmsAreBlind", "provenance": { "record_ids": [ "benchmark_vlmsareblind" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_biological_knowledge", "id": "benchmark_vmqa_vct", "name": "VMQA/VCT", "provenance": { "record_ids": [ "benchmark_vmqa_vct" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_voicebench", "name": "VoiceBench", "provenance": { "record_ids": [ "benchmark_voicebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_voicebench_commoneval", "name": "VoiceBench CommonEval", "provenance": { "record_ids": [ "benchmark_voicebench_commoneval" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "VQA v2" ], "category_id": "category_multimodal_understanding", "id": "benchmark_vqav2", "name": "VQAv2", "provenance": { "record_ids": [ "benchmark_vqav2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_vsi_bench", "name": "VSI-Bench", "provenance": { "record_ids": [ "benchmark_vsi_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_vsr", "name": "VSR", "provenance": { "record_ids": [ "benchmark_vsr" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_mathematics", "id": "benchmark_we_math", "name": "We-Math", "provenance": { "record_ids": [ "benchmark_we_math" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_computer_use", "id": "benchmark_webarena", "name": "WebArena", "provenance": { "record_ids": [ "benchmark_webarena" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Described as the verified release of WebArena" ], "category_id": "category_computer_use", "id": "benchmark_webarena_verified", "name": "WebArena-Verified", "provenance": { "record_ids": [ "benchmark_webarena_verified" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "WebDev Arena Leaderboard" ], "category_id": "category_coding", "id": "benchmark_webdev_arena", "name": "WebDev Arena", "provenance": { "record_ids": [ "benchmark_webdev_arena" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_computer_use", "id": "benchmark_webvoyager", "name": "WebVoyager", "provenance": { "record_ids": [ "benchmark_webvoyager" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_wenetspeech", "name": "WenetSpeech", "provenance": { "record_ids": [ "benchmark_wenetspeech" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_tool_use_and_agents", "id": "benchmark_widesearch", "name": "WideSearch", "provenance": { "record_ids": [ "benchmark_widesearch" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Wikipedia-en" ], "category_id": "category_text_quality", "id": "benchmark_wikipedia_english", "name": "Wikipedia English", "provenance": { "record_ids": [ "benchmark_wikipedia_english" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_summarization", "id": "benchmark_wikisummary", "name": "WikiSummary", "provenance": { "record_ids": [ "benchmark_wikisummary" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "(InThe)WildChat", "Wildchat Non-toxic", "Wildchat Toxic" ], "category_id": "category_safety_and_alignment", "id": "benchmark_wildchat", "name": "WildChat", "provenance": { "record_ids": [ "benchmark_wildchat" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_audio_and_speech", "id": "benchmark_wildspeech_bench", "name": "WildSpeech-Bench", "provenance": { "record_ids": [ "benchmark_wildspeech_bench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Winobias 1_2" ], "category_id": "category_safety_and_responsibility", "id": "benchmark_winobias_type_1_2", "name": "WinoBias Type 1-2", "provenance": { "record_ids": [ "benchmark_winobias_type_1_2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "Winobias 2_2" ], "category_id": "category_safety_and_responsibility", "id": "benchmark_winobias_type_2_2", "name": "WinoBias Type 2-2", "provenance": { "record_ids": [ "benchmark_winobias_type_2_2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_safety_and_responsibility", "id": "benchmark_winogender", "name": "WinoGender", "provenance": { "record_ids": [ "benchmark_winogender" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_commonsense_reasoning", "id": "benchmark_winogrande", "name": "WinoGrande", "provenance": { "record_ids": [ "benchmark_winogrande" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_wmdp_bio", "name": "WMDP-Bio", "provenance": { "record_ids": [ "benchmark_wmdp_bio" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_wmdp_biology", "name": "WMDP Biology", "provenance": { "record_ids": [ "benchmark_wmdp_biology" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_wmdp_chem", "name": "WMDP-Chem", "provenance": { "record_ids": [ "benchmark_wmdp_chem" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_health_and_biology", "id": "benchmark_wmdp_chemistry", "name": "WMDP Chemistry", "provenance": { "record_ids": [ "benchmark_wmdp_chemistry" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_robustness_and_safety", "id": "benchmark_wmdp_cyber", "name": "WMDP-Cyber", "provenance": { "record_ids": [ "benchmark_wmdp_cyber" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_wmt23", "name": "WMT23", "provenance": { "record_ids": [ "benchmark_wmt23" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_wmt24", "name": "WMT24++", "provenance": { "record_ids": [ "benchmark_wmt24" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_agents_and_tool_use", "id": "benchmark_workspacebench", "name": "WorkSpaceBench", "provenance": { "record_ids": [ "benchmark_workspacebench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_commonsense_reasoning", "id": "benchmark_worldsense", "name": "WorldSense", "provenance": { "record_ids": [ "benchmark_worldsense" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_vision_language_understanding", "id": "benchmark_worldvqa", "name": "WorldVQA", "provenance": { "record_ids": [ "benchmark_worldvqa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_human_preference", "id": "benchmark_writingbench", "name": "WritingBench", "provenance": { "record_ids": [ "benchmark_writingbench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_xcopa", "name": "XCOPA", "provenance": { "record_ids": [ "benchmark_xcopa" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_xquad", "name": "XQuAD", "provenance": { "record_ids": [ "benchmark_xquad" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [ "XSTest evaluations" ], "category_id": "category_safety_and_alignment", "id": "benchmark_xstest", "name": "XSTest", "provenance": { "record_ids": [ "benchmark_xstest" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_xstorycloze", "name": "XStoryCloze", "provenance": { "record_ids": [ "benchmark_xstorycloze" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_summarization", "id": "benchmark_xsum", "name": "XSum", "provenance": { "record_ids": [ "benchmark_xsum" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multilingual", "id": "benchmark_xwinograd", "name": "XWinograd", "provenance": { "record_ids": [ "benchmark_xwinograd" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_multimodal_understanding", "id": "benchmark_youcook2", "name": "YouCook2", "provenance": { "record_ids": [ "benchmark_youcook2" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_knowledge_and_reasoning", "id": "benchmark_zebralogic", "name": "ZebraLogic", "provenance": { "record_ids": [ "benchmark_zebralogic" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_image_understanding", "id": "benchmark_zerobench", "name": "ZeroBench", "provenance": { "record_ids": [ "benchmark_zerobench" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } }, { "aliases": [], "category_id": "category_general_capability", "id": "benchmark_zerobench_sub", "name": "ZEROBench-Sub", "provenance": { "record_ids": [ "benchmark_zerobench_sub" ], "record_type": "benchmarks", "source_file": "data/catalog.yaml" } } ], "canonical_definitions": [ { "algorithm": { "comparable_scope": "same_lineage_modality_and_use_class", "kind": "present_after_immediately_prior_presence", "omission_count": 0 }, "cautions": [ "Continuity describes public reporting only." ], "description": "The benchmark is present in reviewed public reporting for this release and the immediately preceding comparable reviewed release also reported it.", "examples": [ "Consecutive comparable releases both name the benchmark in reviewed reporting." ], "id": "continued", "label": "Continued", "provenance": { "record_ids": [ "continued" ], "record_type": "definition", "source_file": "data/definitions.yaml" }, "version": "1.0.0" }, { "algorithm": { "comparable_scope": "same_lineage_modality_and_use_class", "kind": "consecutive_reviewed_omissions", "omission_count": 1 }, "cautions": [ "Dormant is not a claim about private evaluation or benchmark quality." ], "description": "The first consecutive comparable reviewed successor omission after a qualifying occurrence.", "examples": [ "One fully reviewed comparable successor omits the benchmark." ], "id": "dormant_in_public_reporting", "label": "Dormant in public reporting", "provenance": { "record_ids": [ "dormant_in_public_reporting" ], "record_type": "definition", "source_file": "data/definitions.yaml" }, "version": "1.0.0" }, { "algorithm": { "comparable_scope": "same_lineage_modality_and_use_class", "kind": "present_without_prior_occurrence", "omission_count": 0 }, "cautions": [ "This does not establish the first internal use of the benchmark." ], "description": "The benchmark is present in reviewed public reporting for this release and has no earlier qualifying occurrence in the same comparable lineage.", "examples": [ "A benchmark first appears in the reviewed evidence for a lineage release." ], "id": "first_reported", "label": "First reported", "provenance": { "record_ids": [ "first_reported" ], "record_type": "definition", "source_file": "data/definitions.yaml" }, "version": "1.0.0" }, { "algorithm": { "comparable_scope": "same_lineage_modality_and_use_class", "kind": "coverage_or_comparability_uncertain", "omission_count": 0 }, "cautions": [ "The system must not advance dormant or outdated while evidence is insufficient." ], "description": "Source coverage, successor order, lineage identity, or comparability is not strong enough to derive an omission status.", "examples": [ "A release has incomplete source review, so omission counting pauses." ], "id": "insufficient_evidence", "label": "Insufficient evidence", "provenance": { "record_ids": [ "insufficient_evidence" ], "record_type": "definition", "source_file": "data/definitions.yaml" }, "version": "1.0.0" }, { "algorithm": { "comparable_scope": "same_lineage_modality_and_use_class", "kind": "absent_after_prior_presence", "omission_count": 1 }, "cautions": [ "Absence from reviewed public reporting is not evidence that an evaluation was not run." ], "description": "A comparable successor with complete source review does not name a benchmark previously reported in that lineage.", "examples": [ "The next comparable reviewed release omits a previously reported benchmark." ], "id": "not_reported_in_reviewed_successor", "label": "Not reported in reviewed successor", "provenance": { "record_ids": [ "not_reported_in_reviewed_successor" ], "record_type": "definition", "source_file": "data/definitions.yaml" }, "version": "1.0.0" }, { "algorithm": { "comparable_scope": "same_lineage_modality_and_use_class", "kind": "consecutive_reviewed_omissions", "omission_count": 2 }, "cautions": [ "Outdated describes a public-reporting sequence and does not mean objectively obsolete." ], "description": "The second and each later consecutive comparable reviewed successor omission after a qualifying occurrence.", "examples": [ "Two consecutive fully reviewed comparable successors omit the benchmark." ], "id": "outdated_in_public_reporting", "label": "Outdated in public reporting", "provenance": { "record_ids": [ "outdated_in_public_reporting" ], "record_type": "definition", "source_file": "data/definitions.yaml" }, "version": "1.0.0" }, { "algorithm": { "comparable_scope": "same_lineage_modality_and_use_class", "kind": "evidence_gap_requires_exclusion", "omission_count": 0 }, "cautions": [ "Quarantine records the gap without predicting what missing evidence would show." ], "description": "A candidate or evidence record is withheld from verified occurrences because a concrete evidence gap remains.", "examples": [ "Public material does not identify the model precisely enough for a verified occurrence." ], "id": "quarantined", "label": "Quarantined", "provenance": { "record_ids": [ "quarantined" ], "record_type": "definition", "source_file": "data/definitions.yaml" }, "version": "1.0.0" }, { "algorithm": { "comparable_scope": "same_lineage_modality_and_use_class", "kind": "present_after_prior_omission", "omission_count": 0 }, "cautions": [ "Returning does not imply that evaluation use stopped during the omission." ], "description": "The benchmark is present after at least one comparable reviewed successor omitted it from public reporting.", "examples": [ "A later comparable release reports a benchmark after an intervening reviewed omission." ], "id": "returning", "label": "Returning", "provenance": { "record_ids": [ "returning" ], "record_type": "definition", "source_file": "data/definitions.yaml" }, "version": "1.0.0" } ], "categories": [ { "aliases": [], "id": "category_abstract_reasoning", "name": "abstract reasoning", "provenance": { "record_ids": [ "category_abstract_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_agentic_and_tool_use", "name": "Agentic and tool use", "provenance": { "record_ids": [ "category_agentic_and_tool_use" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_agentic_misalignment", "name": "agentic misalignment", "provenance": { "record_ids": [ "category_agentic_misalignment" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_agentic_search", "name": "agentic search", "provenance": { "record_ids": [ "category_agentic_search" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_agentic_tool_use", "name": "agentic tool use", "provenance": { "record_ids": [ "category_agentic_tool_use" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_agents_and_tool_use", "name": "Agents and tool use", "provenance": { "record_ids": [ "category_agents_and_tool_use" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_ai_research_and_development", "name": "ai research and development", "provenance": { "record_ids": [ "category_ai_research_and_development" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_audio_and_speech", "name": "Audio and speech", "provenance": { "record_ids": [ "category_audio_and_speech" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_autonomous_cybersecurity", "name": "autonomous cybersecurity", "provenance": { "record_ids": [ "category_autonomous_cybersecurity" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_autonomy_and_software_engineering", "name": "autonomy and software engineering", "provenance": { "record_ids": [ "category_autonomy_and_software_engineering" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biological_cloning_workflows", "name": "biological cloning workflows", "provenance": { "record_ids": [ "category_biological_cloning_workflows" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biological_figure_interpretation", "name": "biological figure interpretation", "provenance": { "record_ids": [ "category_biological_figure_interpretation" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biological_knowledge", "name": "biological knowledge", "provenance": { "record_ids": [ "category_biological_knowledge" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biological_laboratory_skills", "name": "biological laboratory skills", "provenance": { "record_ids": [ "category_biological_laboratory_skills" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biological_protocol_understanding", "name": "biological protocol understanding", "provenance": { "record_ids": [ "category_biological_protocol_understanding" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biological_risk", "name": "biological risk", "provenance": { "record_ids": [ "category_biological_risk" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biological_risk_agentic_evaluation", "name": "biological risk agentic evaluation", "provenance": { "record_ids": [ "category_biological_risk_agentic_evaluation" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biological_risk_knowledge", "name": "biological risk knowledge", "provenance": { "record_ids": [ "category_biological_risk_knowledge" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biological_sequence_reasoning", "name": "biological sequence reasoning", "provenance": { "record_ids": [ "category_biological_sequence_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biology_multimodal", "name": "biology multimodal", "provenance": { "record_ids": [ "category_biology_multimodal" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_biomedical_question_answering", "name": "biomedical question answering", "provenance": { "record_ids": [ "category_biomedical_question_answering" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_chart_question_answering", "name": "chart question answering", "provenance": { "record_ids": [ "category_chart_question_answering" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_code_generation", "name": "code generation", "provenance": { "record_ids": [ "category_code_generation" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_coding", "name": "Coding", "provenance": { "record_ids": [ "category_coding" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_commonsense_and_science_reasoning", "name": "commonsense and science reasoning", "provenance": { "record_ids": [ "category_commonsense_and_science_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_commonsense_reasoning", "name": "commonsense reasoning", "provenance": { "record_ids": [ "category_commonsense_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_competition_mathematics", "name": "competition mathematics", "provenance": { "record_ids": [ "category_competition_mathematics" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_computational_biology", "name": "computational biology", "provenance": { "record_ids": [ "category_computational_biology" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_computer_use", "name": "Computer use", "provenance": { "record_ids": [ "category_computer_use" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [ "cyber" ], "id": "category_cybersecurity", "name": "Cybersecurity", "provenance": { "record_ids": [ "category_cybersecurity" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_deep_research", "name": "deep research", "provenance": { "record_ids": [ "category_deep_research" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_document_reasoning", "name": "document reasoning", "provenance": { "record_ids": [ "category_document_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_document_visual_question_answering", "name": "document visual question answering", "provenance": { "record_ids": [ "category_document_visual_question_answering" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [ "grounding" ], "id": "category_factuality", "name": "Factuality", "provenance": { "record_ids": [ "category_factuality" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_finance", "name": "finance", "provenance": { "record_ids": [ "category_finance" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_finance_agent", "name": "finance agent", "provenance": { "record_ids": [ "category_finance_agent" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_general_capability", "name": "General capability", "provenance": { "record_ids": [ "category_general_capability" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_general_knowledge", "name": "General knowledge", "provenance": { "record_ids": [ "category_general_knowledge" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_general_reasoning", "name": "General reasoning", "provenance": { "record_ids": [ "category_general_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_general_reasoning_and_knowledge", "name": "general reasoning and knowledge", "provenance": { "record_ids": [ "category_general_reasoning_and_knowledge" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_health_and_biology", "name": "Health and biology", "provenance": { "record_ids": [ "category_health_and_biology" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_health_and_science", "name": "Health and science", "provenance": { "record_ids": [ "category_health_and_science" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [ "medical" ], "id": "category_healthcare", "name": "healthcare", "provenance": { "record_ids": [ "category_healthcare" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_honesty", "name": "honesty", "provenance": { "record_ids": [ "category_honesty" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_honesty_and_hallucination", "name": "honesty and hallucination", "provenance": { "record_ids": [ "category_honesty_and_hallucination" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_human_preference", "name": "Human preference", "provenance": { "record_ids": [ "category_human_preference" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_image_understanding", "name": "Image understanding", "provenance": { "record_ids": [ "category_image_understanding" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_instruction_and_preference", "name": "Instruction And Preference", "provenance": { "record_ids": [ "category_instruction_and_preference" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_instruction_following", "name": "Instruction following", "provenance": { "record_ids": [ "category_instruction_following" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_internal_ai_research_autonomy", "name": "internal ai research autonomy", "provenance": { "record_ids": [ "category_internal_ai_research_autonomy" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_internal_factuality", "name": "internal factuality", "provenance": { "record_ids": [ "category_internal_factuality" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_interpretability", "name": "interpretability", "provenance": { "record_ids": [ "category_interpretability" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_interpretability_control", "name": "interpretability control", "provenance": { "record_ids": [ "category_interpretability_control" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_knowledge_and_reasoning", "name": "Knowledge and reasoning", "provenance": { "record_ids": [ "category_knowledge_and_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_knowledge_reasoning", "name": "Knowledge Reasoning", "provenance": { "record_ids": [ "category_knowledge_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_language_modeling", "name": "Language Modeling", "provenance": { "record_ids": [ "category_language_modeling" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_life_sciences", "name": "life sciences", "provenance": { "record_ids": [ "category_life_sciences" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_long_context", "name": "Long context", "provenance": { "record_ids": [ "category_long_context" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_long_context_reading_comprehension", "name": "long context reading comprehension", "provenance": { "record_ids": [ "category_long_context_reading_comprehension" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_long_context_software_engineering", "name": "long context software engineering", "provenance": { "record_ids": [ "category_long_context_software_engineering" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_long_context_understanding", "name": "Long-context understanding", "provenance": { "record_ids": [ "category_long_context_understanding" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_long_horizon_agents", "name": "long horizon agents", "provenance": { "record_ids": [ "category_long_horizon_agents" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_mathematical_reasoning", "name": "mathematical reasoning", "provenance": { "record_ids": [ "category_mathematical_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [ "math" ], "id": "category_mathematics", "name": "mathematics", "provenance": { "record_ids": [ "category_mathematics" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_mathematics_and_science", "name": "Mathematics and science", "provenance": { "record_ids": [ "category_mathematics_and_science" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_medical_knowledge", "name": "medical knowledge", "provenance": { "record_ids": [ "category_medical_knowledge" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_medical_question_answering", "name": "medical question answering", "provenance": { "record_ids": [ "category_medical_question_answering" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_medical_reasoning", "name": "medical reasoning", "provenance": { "record_ids": [ "category_medical_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_mixed_reasoning", "name": "mixed reasoning", "provenance": { "record_ids": [ "category_mixed_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [ "cross-lingual", "translation" ], "id": "category_multilingual", "name": "multilingual", "provenance": { "record_ids": [ "category_multilingual" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_multilingual_knowledge", "name": "multilingual knowledge", "provenance": { "record_ids": [ "category_multilingual_knowledge" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_multimodal", "name": "multimodal", "provenance": { "record_ids": [ "category_multimodal" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_multimodal_reasoning", "name": "multimodal reasoning", "provenance": { "record_ids": [ "category_multimodal_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_multimodal_software_engineering", "name": "multimodal software engineering", "provenance": { "record_ids": [ "category_multimodal_software_engineering" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [ "vision-language" ], "id": "category_multimodal_understanding", "name": "Multimodal understanding", "provenance": { "record_ids": [ "category_multimodal_understanding" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_multimodal_virology_knowledge", "name": "multimodal virology knowledge", "provenance": { "record_ids": [ "category_multimodal_virology_knowledge" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_office_productivity", "name": "office productivity", "provenance": { "record_ids": [ "category_office_productivity" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_professional_tasks", "name": "professional tasks", "provenance": { "record_ids": [ "category_professional_tasks" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_professional_work", "name": "Professional work", "provenance": { "record_ids": [ "category_professional_work" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_prompt_injection", "name": "prompt injection", "provenance": { "record_ids": [ "category_prompt_injection" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_question_answering", "name": "Question answering", "provenance": { "record_ids": [ "category_question_answering" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_reading_comprehension", "name": "reading comprehension", "provenance": { "record_ids": [ "category_reading_comprehension" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_reading_comprehension_and_discrete_reasoning", "name": "reading comprehension and discrete reasoning", "provenance": { "record_ids": [ "category_reading_comprehension_and_discrete_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_reasoning", "name": "Reasoning", "provenance": { "record_ids": [ "category_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [ "knowledge" ], "id": "category_reasoning_and_knowledge", "name": "Reasoning and knowledge", "provenance": { "record_ids": [ "category_reasoning_and_knowledge" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_research_engineering", "name": "research engineering", "provenance": { "record_ids": [ "category_research_engineering" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_reward_hacking", "name": "reward hacking", "provenance": { "record_ids": [ "category_reward_hacking" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_robustness_and_safety", "name": "Robustness and safety", "provenance": { "record_ids": [ "category_robustness_and_safety" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_sabotage_capability", "name": "sabotage capability", "provenance": { "record_ids": [ "category_sabotage_capability" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_safeguard_evasion", "name": "safeguard evasion", "provenance": { "record_ids": [ "category_safeguard_evasion" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_safety_and_alignment", "name": "Safety and alignment", "provenance": { "record_ids": [ "category_safety_and_alignment" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [ "responsible AI" ], "id": "category_safety_and_responsibility", "name": "Safety and responsibility", "provenance": { "record_ids": [ "category_safety_and_responsibility" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_science_diagram_reasoning", "name": "science diagram reasoning", "provenance": { "record_ids": [ "category_science_diagram_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_software_engineering", "name": "software engineering", "provenance": { "record_ids": [ "category_software_engineering" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_software_reliability", "name": "software reliability", "provenance": { "record_ids": [ "category_software_reliability" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_standardized_analytical_writing", "name": "standardized analytical writing", "provenance": { "record_ids": [ "category_standardized_analytical_writing" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_standardized_legal_knowledge", "name": "standardized legal knowledge", "provenance": { "record_ids": [ "category_standardized_legal_knowledge" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_standardized_legal_reasoning", "name": "standardized legal reasoning", "provenance": { "record_ids": [ "category_standardized_legal_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_standardized_quantitative_reasoning", "name": "standardized quantitative reasoning", "provenance": { "record_ids": [ "category_standardized_quantitative_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_standardized_verbal_reasoning", "name": "standardized verbal reasoning", "provenance": { "record_ids": [ "category_standardized_verbal_reasoning" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_summarization", "name": "Summarization", "provenance": { "record_ids": [ "category_summarization" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_text_quality", "name": "Text quality", "provenance": { "record_ids": [ "category_text_quality" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_tool_use", "name": "Tool use", "provenance": { "record_ids": [ "category_tool_use" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_tool_use_and_agents", "name": "Tool use and agents", "provenance": { "record_ids": [ "category_tool_use_and_agents" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_video_understanding", "name": "Video understanding", "provenance": { "record_ids": [ "category_video_understanding" ], "record_type": "categories", "source_file": "data/catalog.yaml" } }, { "aliases": [], "id": "category_vision_language_understanding", "name": "Vision-language understanding", "provenance": { "record_ids": [ "category_vision_language_understanding" ], "record_type": "categories", "source_file": "data/catalog.yaml" } } ], "corpus": { "id": "frontier_benchmark_observatory_2026_09_01", "publication_window": { "end": "2026-09-01", "start": "2024-01-01" }, "retrieval_date": "2026-09-01", "title": "Frontier Benchmark Observatory bounded six-lab corpus" }, "coverage": [ { "id": "coverage_anthropic_20240304_introducing_the_next_generation_of_claude", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20240304_introducing_the_next_generation_of_claude", "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d" ] }, { "id": "coverage_anthropic_20240621_claude_3_5_sonnet", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20240621_claude_3_5_sonnet", "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03" ] }, { "id": "coverage_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c" ] }, { "id": "coverage_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6" ] }, { "id": "coverage_anthropic_20250522_introducing_claude_4", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20250522_introducing_claude_4" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20250522_introducing_claude_4", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20250522_introducing_claude_4" ] }, { "id": "coverage_anthropic_20250805_claude_opus_4_1", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20250805_claude_opus_4_1" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20250805_claude_opus_4_1", "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2" ] }, { "id": "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e" ] }, { "id": "coverage_anthropic_20251015_introducing_claude_haiku_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141" ] }, { "id": "coverage_anthropic_20251124_introducing_claude_opus_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20251124_introducing_claude_opus_4_5", "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8" ] }, { "id": "coverage_anthropic_20260205_introducing_claude_opus_4_6", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20260205_introducing_claude_opus_4_6", "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c" ] }, { "id": "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20260217_introducing_claude_sonnet_4_6", "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6" ] }, { "id": "coverage_anthropic_20260416_introducing_claude_opus_4_7", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20260416_introducing_claude_opus_4_7", "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2" ] }, { "id": "coverage_anthropic_20260528_introducing_claude_opus_4_8", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20260528_introducing_claude_opus_4_8", "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f" ] }, { "id": "coverage_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca" ] }, { "id": "coverage_anthropic_20260630_introducing_claude_sonnet_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20260630_introducing_claude_sonnet_5", "source_anthropic_technical_0ce49aa473c3037455b45579980fd0b55708c1066df3f5f99f92bb3169d9b9b7", "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed" ] }, { "id": "coverage_anthropic_20260724_introducing_claude_opus_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_anthropic_launch_anthropic_20260724_introducing_claude_opus_5", "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b" ] }, { "id": "coverage_deepseek_20240517_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20240517_deepseek_chat" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20240517_deepseek_chat", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_launch_deepseek_20240517_deepseek_chat" ] }, { "id": "coverage_deepseek_20240614_deepseek_coder", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20240614_deepseek_coder" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20240614_deepseek_coder", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_launch_deepseek_20240614_deepseek_coder" ] }, { "id": "coverage_deepseek_20240628_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20240628_deepseek_chat" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20240628_deepseek_chat", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_launch_deepseek_20240628_deepseek_chat" ] }, { "id": "coverage_deepseek_20240724_deepseek_coder", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20240724_deepseek_coder" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20240724_deepseek_coder", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_launch_deepseek_20240724_deepseek_coder" ] }, { "id": "coverage_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_hf_v25", "source_deepseek_source_launch_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ] }, { "id": "coverage_deepseek_20241120_deepseek_r1_lite_preview", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20241120_deepseek_r1_lite_preview" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20241120_deepseek_r1_lite_preview", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_launch_deepseek_20241120_deepseek_r1_lite_preview" ] }, { "id": "coverage_deepseek_20241210_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20241210_deepseek_chat", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_hf_v25_1210", "source_deepseek_source_launch_deepseek_20241210_deepseek_chat" ] }, { "id": "coverage_deepseek_20241226_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20241226_deepseek_chat" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20241226_deepseek_chat", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_launch_deepseek_20241226_deepseek_chat", "source_deepseek_source_repo_v3" ] }, { "id": "coverage_deepseek_20250120_deepseek_reasoner", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20250120_deepseek_reasoner" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_launch_deepseek_20250120_deepseek_reasoner", "source_deepseek_source_report_r1" ] }, { "id": "coverage_deepseek_20250324_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20250324_deepseek_chat", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_hf_v3_0324", "source_deepseek_source_launch_deepseek_20250324_deepseek_chat" ] }, { "id": "coverage_deepseek_20250528_deepseek_reasoner", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_hf_r1_0528", "source_deepseek_source_launch_deepseek_20250528_deepseek_reasoner" ] }, { "id": "coverage_deepseek_20250821_deepseek_v3_1", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_hf_v31", "source_deepseek_source_launch_deepseek_20250821_deepseek_v3_1" ] }, { "id": "coverage_deepseek_20250922_deepseek_v3_1_terminus", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_hf_v31_terminus", "source_deepseek_source_launch_deepseek_20250922_deepseek_v3_1_terminus" ] }, { "id": "coverage_deepseek_20250929_deepseek_v3_2_exp", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_hf_v32_exp", "source_deepseek_source_launch_deepseek_20250929_deepseek_v3_2_exp", "source_deepseek_source_report_v32_exp" ] }, { "id": "coverage_deepseek_20251201_deepseek_v3_2", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_hf_v32", "source_deepseek_source_launch_deepseek_20251201_deepseek_v3_2", "source_deepseek_source_report_v32" ] }, { "id": "coverage_deepseek_20251201_deepseek_v3_2_speciale", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_hf_v32_speciale", "source_deepseek_source_launch_deepseek_20251201_deepseek_v3_2_speciale", "source_deepseek_source_report_v32" ] }, { "id": "coverage_deepseek_20260424_deepseek_v4", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20260424_deepseek_v4" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20260424_deepseek_v4", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_hf_v4", "source_deepseek_source_launch_deepseek_20260424_deepseek_v4" ] }, { "id": "coverage_deepseek_20260731_deepseek_v4_flash_update", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update" ] }, { "id": "coverage_deepseek_20260813_deepseek_v4_pro_update", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update" ] }, { "id": "coverage_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "lab_id": "deepseek", "provenance": { "record_ids": [ "coverage_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ] }, { "id": "coverage_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20240208_the_next_chapter_of_our_gemini_era" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20240208_the_next_chapter_of_our_gemini_era" ] }, { "id": "coverage_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20240215_our_next_generation_model_gemini_1_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "source_google_deepmind_support_gemini_1_5_report" ] }, { "id": "coverage_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "source_google_deepmind_support_gemma_report" ] }, { "id": "coverage_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "source_google_deepmind_support_gemini_1_5_report" ] }, { "id": "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ] }, { "id": "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ] }, { "id": "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ] }, { "id": "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ] }, { "id": "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ] }, { "id": "coverage_google_deepmind_20250312_introducing_gemma_3", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250312_introducing_gemma_3", "source_google_deepmind_support_gemma_3_report" ] }, { "id": "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ] }, { "id": "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250417_introducing_gemini_2_5_flash" ] }, { "id": "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ] }, { "id": "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ] }, { "id": "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ] }, { "id": "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_google_deepmind_support_gemini_2_5_flash_card", "source_google_deepmind_support_gemini_2_5_pro_card" ] }, { "id": "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ] }, { "id": "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ] }, { "id": "coverage_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai" ] }, { "id": "coverage_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo" ] }, { "id": "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ] }, { "id": "coverage_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250801_try_deep_think_in_the_gemini_app" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "source_google_deepmind_support_gemini_2_5_deep_think_card" ] }, { "id": "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ] }, { "id": "coverage_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "source_google_deepmind_support_vaultgemma_report" ] }, { "id": "coverage_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "source_google_deepmind_support_computer_use_eval_info", "source_google_deepmind_support_computer_use_system_card" ] }, { "id": "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_google_deepmind_support_gemini_3_eval_methodology", "source_google_deepmind_support_gemini_3_model_card" ] }, { "id": "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ] }, { "id": "coverage_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w" ] }, { "id": "coverage_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ] }, { "id": "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ] }, { "id": "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ] }, { "id": "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "source_google_deepmind_support_gemma_4_model_card" ] }, { "id": "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ] }, { "id": "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ] }, { "id": "coverage_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation" ] }, { "id": "coverage_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash" ] }, { "id": "coverage_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber" ] }, { "id": "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ] }, { "id": "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_google_deepmind_support_gemini_3_7_flash_model_card" ] }, { "id": "coverage_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_llama_3", "source_meta_tech_llama_3_card", "source_meta_tech_llama_3_eval" ] }, { "id": "coverage_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_llama_3_1", "source_meta_paper_llama_3_herd", "source_meta_tech_llama_3_1_card", "source_meta_tech_llama_3_1_eval" ] }, { "id": "coverage_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_llama_3_2", "source_meta_tech_llama_3_2_eval", "source_meta_tech_llama_3_2_text_card", "source_meta_tech_llama_3_2_vision_card" ] }, { "id": "coverage_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_llama_quantized", "source_meta_tech_llama_3_2_text_card" ] }, { "id": "coverage_meta_20241219_the_future_of_ai_built_with_llama", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_llama_3_3" ] }, { "id": "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_llama_4", "source_meta_llama_downloads", "source_meta_tech_llama_4_card" ] }, { "id": "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_llamacon" ] }, { "id": "coverage_meta_20260408_introducing_muse_spark", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_20260408_introducing_muse_spark" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_20260408_introducing_muse_spark", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_muse_spark", "source_meta_report_muse_methodology", "source_meta_report_muse_safety" ] }, { "id": "coverage_meta_20260709_introducing_muse_spark_1_1", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_muse_spark_1_1", "source_meta_report_muse_1_1", "source_meta_report_muse_1_1_methodology" ] }, { "id": "coverage_meta_derived_20240618_meta_chameleon", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_derived_20240618_meta_chameleon" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_derived_20240618_meta_chameleon", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_fair_june", "source_meta_paper_chameleon" ] }, { "id": "coverage_meta_derived_20240618_multi_token_prediction", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_derived_20240618_multi_token_prediction" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_fair_june", "source_meta_paper_mtp" ] }, { "id": "coverage_meta_derived_20241018_code_llama_checkpoints", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_derived_20241018_code_llama_checkpoints" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_derived_20241018_code_llama_checkpoints", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_fair_october", "source_meta_paper_layerskip", "source_meta_repo_layerskip" ] }, { "id": "coverage_meta_derived_20241018_layer_skip_llama_checkpoints", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_fair_october", "source_meta_paper_layerskip", "source_meta_repo_layerskip" ] }, { "id": "coverage_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_fair_december", "source_meta_paper_blt" ] }, { "id": "coverage_meta_derived_20241212_meta_large_concept_models", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_derived_20241212_meta_large_concept_models" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_fair_december", "source_meta_paper_lcm" ] }, { "id": "coverage_meta_derived_20250417_meta_perception_language_model", "lab_id": "meta", "provenance": { "record_ids": [ "coverage_meta_derived_20250417_meta_perception_language_model" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_meta_launch_plm", "source_meta_paper_plm" ] }, { "id": "coverage_openai_20240125_new_embedding_models_and_api_updates", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20240125_new_embedding_models_and_api_updates" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20240125_new_embedding_models_and_api_updates", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20240125_new_embedding_models_and_api_updates" ] }, { "id": "coverage_openai_20240513_hello_gpt_4o", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20240513_hello_gpt_4o" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20240513_hello_gpt_4o", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20240513_hello_gpt_4o" ] }, { "id": "coverage_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ] }, { "id": "coverage_openai_20240912_introducing_openai_o1_preview", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20240912_introducing_openai_o1_preview" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_openai_source_openai_20240912_introducing_openai_o1_preview" ] }, { "id": "coverage_openai_20240912_openai_o1_mini", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20240912_openai_o1_mini" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20240912_openai_o1_mini", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_openai_source_openai_20240912_openai_o1_mini" ] }, { "id": "coverage_openai_20241217_openai_o1_and_new_tools_for_developers", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers" ] }, { "id": "coverage_openai_20250123_computer_using_agent", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20250123_computer_using_agent" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20250123_computer_using_agent", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20250123_computer_using_agent" ] }, { "id": "coverage_openai_20250131_openai_o3_mini", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20250131_openai_o3_mini" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20250131_openai_o3_mini", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20250131_openai_o3_mini" ] }, { "id": "coverage_openai_20250227_introducing_gpt_4_5", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20250227_introducing_gpt_4_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20250227_introducing_gpt_4_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20250227_introducing_gpt_4_5" ] }, { "id": "coverage_openai_20250414_introducing_gpt_4_1_in_the_api", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api" ] }, { "id": "coverage_openai_20250416_introducing_openai_o3_and_o4_mini", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini" ] }, { "id": "coverage_openai_20250516_introducing_codex", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20250516_introducing_codex" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20250516_introducing_codex", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20250516_introducing_codex" ] }, { "id": "coverage_openai_20250805_introducing_gpt_oss", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20250805_introducing_gpt_oss" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20250805_introducing_gpt_oss", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20250805_introducing_gpt_oss" ] }, { "id": "coverage_openai_20250807_introducing_gpt_5", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20250807_introducing_gpt_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20250807_introducing_gpt_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20250807_introducing_gpt_5" ] }, { "id": "coverage_openai_20250915_introducing_upgrades_to_codex", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20250915_introducing_upgrades_to_codex" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20250915_introducing_upgrades_to_codex", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20250915_introducing_upgrades_to_codex" ] }, { "id": "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ] }, { "id": "coverage_openai_20251119_building_more_with_gpt_5_1_codex_max", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20251119_building_more_with_gpt_5_1_codex_max" ] }, { "id": "coverage_openai_20251211_introducing_gpt_5_2", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20251211_introducing_gpt_5_2" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20251211_introducing_gpt_5_2" ] }, { "id": "coverage_openai_20251218_introducing_gpt_5_2_codex", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20251218_introducing_gpt_5_2_codex" ] }, { "id": "coverage_openai_20260205_introducing_gpt_5_3_codex", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260205_introducing_gpt_5_3_codex" ] }, { "id": "coverage_openai_20260212_introducing_gpt_5_3_codex_spark", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260212_introducing_gpt_5_3_codex_spark" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260212_introducing_gpt_5_3_codex_spark", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260212_introducing_gpt_5_3_codex_spark" ] }, { "id": "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ] }, { "id": "coverage_openai_20260305_introducing_gpt_5_4", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260305_introducing_gpt_5_4" ] }, { "id": "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano" ] }, { "id": "coverage_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research" ] }, { "id": "coverage_openai_20260423_introducing_gpt_5_5", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_https_deploymentsafety_openai_com_gpt_5_5", "source_openai_source_openai_20260423_introducing_gpt_5_5" ] }, { "id": "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ] }, { "id": "coverage_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ] }, { "id": "coverage_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ] }, { "id": "coverage_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ] }, { "id": "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ] }, { "id": "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ] }, { "id": "coverage_openai_20260811_daybreak_models_are_now_available_on_aws", "lab_id": "openai", "provenance": { "record_ids": [ "coverage_openai_20260811_daybreak_models_are_now_available_on_aws" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_openai_20260811_daybreak_models_are_now_available_on_aws", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_openai_source_openai_20260811_daybreak_models_are_now_available_on_aws" ] }, { "id": "coverage_qwen_20240123_introducing_qwen_qwen", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240123_introducing_qwen_qwen" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240123_introducing_qwen_qwen", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240123_introducing_qwen_qwen" ] }, { "id": "coverage_qwen_20240125_introducing_qwen_vl_qwen_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl" ] }, { "id": "coverage_qwen_20240204_introducing_qwen1_5_qwen1_5", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5" ] }, { "id": "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ] }, { "id": "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ] }, { "id": "coverage_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ] }, { "id": "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ] }, { "id": "coverage_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428" ] }, { "id": "coverage_qwen_20240606_hello_qwen2_qwen2", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240606_hello_qwen2_qwen2" ] }, { "id": "coverage_qwen_20240807_introducing_qwen2_math_qwen2_math", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math" ] }, { "id": "coverage_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ] }, { "id": "coverage_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ] }, { "id": "coverage_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ] }, { "id": "coverage_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ] }, { "id": "coverage_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ] }, { "id": "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ] }, { "id": "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ] }, { "id": "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ] }, { "id": "coverage_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ] }, { "id": "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ] }, { "id": "coverage_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ] }, { "id": "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ] }, { "id": "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ] }, { "id": "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ] }, { "id": "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ] }, { "id": "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ] }, { "id": "coverage_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ] }, { "id": "coverage_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ] }, { "id": "coverage_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo" ] }, { "id": "coverage_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder" ] }, { "id": "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ] }, { "id": "coverage_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ] }, { "id": "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ] }, { "id": "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ] }, { "id": "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ] }, { "id": "coverage_qwen_20260126_qwen3_max_thinking", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260126_qwen3_max_thinking" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260126_qwen3_max_thinking" ] }, { "id": "coverage_qwen_20260203_qwen3_coder_next", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260203_qwen3_coder_next" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260203_qwen3_coder_next", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260203_qwen3_coder_next" ] }, { "id": "coverage_qwen_20260216_qwen3_5", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260216_qwen3_5" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260216_qwen3_5", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260216_qwen3_5" ] }, { "id": "coverage_qwen_20260319_qwen3_5_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260319_qwen3_5_max_preview" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260319_qwen3_5_max_preview", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260319_qwen3_5_max_preview" ] }, { "id": "coverage_qwen_20260330_qwen3_5_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260330_qwen3_5_omni", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260330_qwen3_5_omni" ] }, { "id": "coverage_qwen_20260402_qwen3_6", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260402_qwen3_6" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260402_qwen3_6", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260402_qwen3_6" ] }, { "id": "coverage_qwen_20260415_qwen3_6_35b_a3b", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b" ] }, { "id": "coverage_qwen_20260418_qwen3_6_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260418_qwen3_6_max_preview" ] }, { "id": "coverage_qwen_20260422_qwen3_6_27b", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260422_qwen3_6_27b", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260422_qwen3_6_27b" ] }, { "id": "coverage_qwen_20260520_qwen3_7", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260520_qwen3_7" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260520_qwen3_7", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260520_qwen3_7" ] }, { "id": "coverage_qwen_20260601_qwen3_7_plus", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260601_qwen3_7_plus" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260601_qwen3_7_plus", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260601_qwen3_7_plus" ] }, { "id": "coverage_qwen_20260803_qwen3_8", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260803_qwen3_8" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260803_qwen3_8", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260803_qwen3_8" ] }, { "id": "coverage_qwen_20260826_qwen3_8_flash_next", "lab_id": "qwen", "provenance": { "record_ids": [ "coverage_qwen_20260826_qwen3_8_flash_next" ], "record_type": "coverage", "source_file": "data/catalog.yaml" }, "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_date": "2026-09-01", "review_status": "complete", "reviewed_source_ids": [ "source_qwen_source_qwen_20260826_qwen3_8_flash_next" ] } ], "definitions_version": "1.0.0", "derived_statuses": [ { "benchmark_id": "benchmark_codeu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240123_introducing_qwen_qwen_benchmark_codeu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_codeu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_codeu", "release_qwen_20240123_introducing_qwen_qwen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_coursera", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240123_introducing_qwen_qwen_benchmark_coursera", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_coursera", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_coursera", "release_qwen_20240123_introducing_qwen_qwen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240123_introducing_qwen_qwen_benchmark_gsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_gsm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_gsm", "release_qwen_20240123_introducing_qwen_qwen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_l_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240123_introducing_qwen_qwen_benchmark_l_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_l_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_l_eval", "release_qwen_20240123_introducing_qwen_qwen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240123_introducing_qwen_qwen_benchmark_needle_in_a_haystack", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_needle_in_a_haystack", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_needle_in_a_haystack", "release_qwen_20240123_introducing_qwen_qwen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quality", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240123_introducing_qwen_qwen_benchmark_quality", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_quality", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_quality", "release_qwen_20240123_introducing_qwen_qwen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_sfiction", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240123_introducing_qwen_qwen_benchmark_sfiction", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_sfiction", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_sfiction", "release_qwen_20240123_introducing_qwen_qwen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toefl", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240123_introducing_qwen_qwen_benchmark_toefl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_toefl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_toefl", "release_qwen_20240123_introducing_qwen_qwen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aitz", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_aitz", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_aitz", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_aitz", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alfred", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_alfred", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_alfred", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_alfred", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_chartqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_chartqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_chartqa", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_docvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_docvqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_docvqa", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_egoschema", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_egoschema", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_egoschema", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_fncall", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_fncall", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_fncall", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_fncall", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gym_cards", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_gym_cards", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_gym_cards", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_gym_cards", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hallbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_hallbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_hallbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_hallbench", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_infographicvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_infovqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_infovqa", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mathvision", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mathvista", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mathvista", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbench_1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_mmbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmbench_1_1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmbench_1_1", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mme", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_mme", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mme", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mme", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmmu", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmstar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmstar", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_mmt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmt_bench", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmvet", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_mmvet", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmvet", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmvet", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mtvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_mtvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mtvqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mtvqa", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mvbench", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocrbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_ocrbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_ocrbench", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_perception_test", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_perception_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_perceptiontest", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_perceptiontest", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_realworldqa", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_textvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_textvqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_textvqa", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_with_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_videomme_with_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_videomme_with_subtitles", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_without_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240125_introducing_qwen_vl_qwen_vl_benchmark_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_videomme_without_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_videomme_without_subtitles", "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_alpacaeval_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_alpacaeval_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_alpacaeval_2_0", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_bbh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_bbh", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_belebele", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_belebele", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_belebele", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_belebele", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_c_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_c_eval", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_cmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_cmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_cmmlu", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_coursera", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_coursera", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_coursera", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_coursera", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_flores", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_flores", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_flores", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_flores", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_gsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_gsm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_gsm", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_gsm8k", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_humaneval", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_m3exams", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_m3exams", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_m3exams", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_m3exams", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mmlu", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_math", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mbpp", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mgsm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mgsm", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_multi", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_mmlu_multi", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mmlu_multi", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mmlu_multi", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mt_bench", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_paws_x", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_paws_x", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_paws_x", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_paws_x", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quality", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_quality", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_quality", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_quality", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_sfiction", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_sfiction", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_sfiction", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_sfiction", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toefl", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_toefl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_toefl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_toefl", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xcopa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_xcopa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xcopa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xcopa", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xstorycloze", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_xstorycloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xstorycloze", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xstorycloze", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xwinograd", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240204_introducing_qwen1_5_qwen1_5_benchmark_xwinograd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xwinograd", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xwinograd", "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240208_the_next_chapter_of_our_gemini_era_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_ultra_1_0", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240208_the_next_chapter_of_our_gemini_era_mmlu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240208_the_next_chapter_of_our_gemini_era_mmlu", "release_google_deepmind_20240208_the_next_chapter_of_our_gemini_era" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-08", "release_id": "release_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240215_our_next_generation_model_gemini_1_5_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_pro_february_preview", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240215_our_next_generation_model_gemini_1_5_mtob", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240215_our_next_generation_model_gemini_1_5_mtob", "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "release_id": "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240215_our_next_generation_model_gemini_1_5_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_pro_february_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "release_id": "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240215_our_next_generation_model_gemini_1_5_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_pro_february_preview", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240215_our_next_generation_model_gemini_1_5_niah", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240215_our_next_generation_model_gemini_1_5_niah", "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "release_id": "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_agieval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_agieval", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_agieval", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_arc_c", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_arc_challenge", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_arc_challenge", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_arc_easy", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_arc_easy", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bigbench_hard", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bigbench_hard", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbq_ambiguous", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_bbq_ambiguous", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bbq_ambiguous", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bbq_ambiguous", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbq_disambiguated", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_bbq_disambiguated", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bbq_disambiguated", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bbq_disambiguated", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bold", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_bold", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bold", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bold", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_boolq", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_boolq", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_commonsenseqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_commonsenseqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_commonsenseqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_commonsenseqa", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_crows_pairs", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_crows_pairs", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_crows_pairs", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_crows_pairs", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_gsm8k", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_gsm8k", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_hellaswag", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_hellaswag", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_human_eval", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_human_eval", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_mmlu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_mmlu", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_math", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_math", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_mbpp", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_mbpp", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_mbpp", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_natural_questions", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_natural_questions", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_natural_questions", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_natural_questions", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openbookqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_openbookqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_openbookqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_openbookqa", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_piqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_piqa", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realtoxicity", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_realtoxicity", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_realtoxicity", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_realtoxicity", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_socialiqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_socialiqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_social_iqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_social_iqa", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toxigen", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_toxigen", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_toxigen", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_toxigen", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_triviaqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_triviaqa", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_truthfulqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_truthfulqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_truthfulqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_truthfulqa", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winobias_type_1_2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_winobias_type_1_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winobias_type_1_2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winobias_type_1_2", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winobias_type_2_2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_winobias_type_2_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winobias_type_2_2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winobias_type_2_2", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogender", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_winogender", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winogender", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winogender", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_benchmark_winogrande", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winogrande", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winogrande", "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_ai2d", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_ai2d", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_ai2d", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_american_mathematics_competition_10", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_10", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_10", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_10", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_american_mathematics_competition_12", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_12", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_12", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_12", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_american_mathematics_competition_8", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_8", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_8", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_apps", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_apps", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_apps", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_apps", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_arc_c", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_arc_challenge", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_arc_challenge", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bbh", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_big_bench_hard", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_big_bench_hard", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bias_benchmark_for_question_answering", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bias_benchmark_for_question_answering", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bioasq", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bioasq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bioasq", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bioasq", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_chartqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_chartqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_chartqa", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_docvqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_docvqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_docvqa", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_drop", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_drop", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_drop", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_easy_medium_qa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_easy_medium_qa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_easy_medium_qa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_easy_medium_qa", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gpqa_diamond", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gre_general_test_analytical_writing", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_analytical_writing", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_analytical_writing", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_analytical_writing", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gre_general_test_quantitative_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_quantitative_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_quantitative_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_quantitative_reasoning", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gre_general_test_verbal_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_verbal_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_verbal_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_verbal_reasoning", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gsm8k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gsm8k", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gsm8k", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_hellaswag", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_hellaswag", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_hellaswag", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_humaneval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_humaneval", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_humaneval", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_law_school_admission_test", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_law_school_admission_test", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_law_school_admission_test", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_law_school_admission_test", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_massive_multitask_language_understanding", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_massive_multitask_language_understanding", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_math", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_math", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mathvista", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mathvista", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mathvista", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mbpp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mostly_basic_python_problems", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mostly_basic_python_problems", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_medmcqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_medmcqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_medmcqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_medmcqa", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mgsm", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multilingual_grade_school_math", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multilingual_grade_school_math", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multilingual_mmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multilingual_mmlu", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_factual", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multi_factual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multi_factual", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multi_factual", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multistate_bar_examination", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multistate_bar_examination", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multistate_bar_examination", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multistate_bar_examination", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_needle_in_a_haystack", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_needle_in_a_haystack", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_needle_in_a_haystack", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_pubmedqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_pubmedqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_pubmedqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_pubmedqa", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quality", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_quality", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_quality", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_quality", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_race_high", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_race_high", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_race_high", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_race_high", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_100q_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_record_100q_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_x_100q_hard", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_x_100q_hard", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_united_states_medical_licensing_examination", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_united_states_medical_licensing_examination", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_united_states_medical_licensing_examination", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_united_states_medical_licensing_examination", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wildchat", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_wildchat", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_wildchat", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_wildchat", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_winogrande", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_winogrande", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_winogrande", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xstest", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_xstest", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_xstest", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_xstest", "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_alpacaeval_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_belebele", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_belebele", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_cmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_coursera", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_coursera", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_flores", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_flores", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_gsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_gsm8k", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_humaneval", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_m3exams", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_m3exams", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_mmlu", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_multi", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_mmlu_multi", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_mt_bench", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_paws_x", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_paws_x", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_quality", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_quality", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_sfiction", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_sfiction", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_toefl", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_toefl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_xcopa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_xcopa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_xstorycloze", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_xstorycloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_xwinograd", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_benchmark_xwinograd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_alpacaeval_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_alpacaeval_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_alpacaeval_2_0", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_bbh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_bbh", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_belebele", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_belebele", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_c_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_c_eval", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_cmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_cmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_cmmlu", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_coursera", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_coursera", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_flores", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_flores", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_gsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_gsm8k", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_humaneval", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_m3exams", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_m3exams", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mmlu", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_math", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mbpp", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_multi", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_mmlu_multi", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mt_bench", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_paws_x", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_paws_x", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_quality", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_quality", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_sfiction", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_sfiction", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_toefl", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_toefl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_xcopa", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_xcopa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_xstorycloze", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_xstorycloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_xwinograd", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_benchmark_xwinograd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_coding", "model_id": "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_humaneval", "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "release_id": "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval_plus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_benchmark_humaneval_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_coding", "model_id": "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_humaneval_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_humaneval_plus", "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "release_id": "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_coding", "model_id": "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_mbpp", "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "release_id": "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp_plus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_benchmark_mbpp_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_coding", "model_id": "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_mbpp_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_mbpp_plus", "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "release_id": "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agieval_english", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_agieval_english", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_agieval_english", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_agieval_english", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_arc_challenge", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_arc_challenge", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_bbh", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_big_bench_hard", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_big_bench_hard", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_boolq", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_boolq", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_boolq", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_commonsenseqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_commonsenseqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_commonsenseqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_commonsenseqa", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_drop", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_drop", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_drop", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_main", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_gpqa_main", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_gpqa_main", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_gpqa_main", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_gsm8k", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_gsm8k", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_humaneval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_humaneval", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_mmlu", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_math", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_math", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quac", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_quac", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_quac", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_quac", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_squad_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_squad_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_squad_v2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_squad_v2", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa_wiki", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_triviaqa_wiki", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_triviaqa_wiki", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_triviaqa_wiki", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava_benchmark_winogrande", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_winogrande", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_winogrande", "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_alpacaeval_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_alpacaeval_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_alpacaeval_2_0", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_arc_c", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_arc_c", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_arc_c", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_bbh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_bbh", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_belebele", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_belebele", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_cmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_coursera", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_coursera", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_flores", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_flores", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_gpqa", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_gsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_gsm8k", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_hellaswag", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_hellaswag", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_hellaswag", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_humaneval", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_m3exams", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_m3exams", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mmlu", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_math", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mbpp", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_multi", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_mmlu_multi", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mt_bench", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_paws_x", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_paws_x", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_quality", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_quality", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_sfiction", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_sfiction", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_theoremqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_theoremqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_theoremqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_theoremqa", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toefl", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_toefl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_xcopa", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_xcopa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_xstorycloze", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_xstorycloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_xwinograd", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_benchmark_xwinograd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428_benchmark_arena", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_max_text_general", "model_id": "model_qwen_model_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428_arena", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428_arena", "release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-11", "release_id": "release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_max_text_general", "model_id": "model_qwen_model_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428_mt_bench", "release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-11", "release_id": "release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_activitynet_qa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_activitynet_qa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_ai2d", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_ai2d", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_amc_2022_2023", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_amc_2022_2023", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_asrob", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_asrob", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_bigbench_hard", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_bigbench_hard", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_better_chartqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_better_chartqa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_blink", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_blink", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_chartqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_chartqa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_chemical_diagram_qa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_chemical_diagram_qa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_docvqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_docvqa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_drop", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_drop", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_dude", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_dude", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_egoschema", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_egoschema", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_functional_math", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_functional_math", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_gpqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_gpqa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_gsm8k", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_gsm8k", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_hellaswag", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_hellaswag", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_hiddenmath", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_hiddenmath", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_human_eval", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_human_eval", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_infographicvqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_infographicvqa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mtob", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mtob", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mmlu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mmlu", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_math", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_math", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mathvista", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mathvista", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mgsm", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mgsm", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mmmu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mmmu", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_natural2code", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_natural2code", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_niah", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_niah", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_openeqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_openeqa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_physics_finals", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_physics_finals", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_realworldqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_realworldqa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_one_hour_videoqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_one_hour_videoqa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_tat_dqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_tat_dqa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_textvqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_textvqa", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vstar_bench", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vstar_bench", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vatex", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vatex", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vatex_zh", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vatex_zh", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vqav2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vqav2", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_wmt23", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_wmt23", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_youcook2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_youcook2", "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240517_deepseek_chat_benchmark_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "model_id": "model_deepseek_model_deepseek_v2_0517", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_0517_ifeval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_0517_ifeval", "release_deepseek_20240517_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-17", "release_id": "release_deepseek_20240517_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alignbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_alignbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_alignbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_alignbench", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_arc_c", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_arc_c", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_arc_c", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_arena_hard", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_arena_hard", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_bbh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_bbh", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_c_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_c_eval", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_cmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_cmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_cmmlu", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_evalplus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_evalplus", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_gpqa", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_gsm8k", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_hellaswag", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_hellaswag", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_hellaswag", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_humaneval", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_ifeval", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_livecodebench", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_m_mmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_m_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_m_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_m_mmlu", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mmlu", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_math", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mbpp", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mgsm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mgsm", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mmlu_pro", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mt_bench", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_exam", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_multi_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_exam", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_exam", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_mathematics", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_multi_mathematics", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_mathematics", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_mathematics", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_translation", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_multi_translation", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_translation", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_translation", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_multi_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_understanding", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_understanding", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multipl_e", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multipl_e", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_theoremqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_theoremqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_theoremqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_theoremqa", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_truthfulqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_truthfulqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_truthfulqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_truthfulqa", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240606_hello_qwen2_qwen2_benchmark_winogrande", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_winogrande", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_winogrande", "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_arc_challenge", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_arc_challenge", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_arc_easy", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_arc_easy", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_arc_easy", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_boolq", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_boolq", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_boolq", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_flickr30k_captions", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_flickr30k_captions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_flickr30k_captions", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_flickr30k_captions", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_gsm8k", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_gsm8k", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_hellaswag", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_hellaswag", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_mmlu", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_math", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_math", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ms_coco_captions", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_ms_coco_captions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_coco_captions", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_coco_captions", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openbookqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_openbookqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_openbookqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_openbookqa", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_piqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_piqa", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_social_iqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_social_iqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_social_iqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_social_iqa", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_vqav2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_vqav2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_vqav2", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_meta_chameleon_benchmark_winogrande", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "model_id": "model_meta_model_meta_chameleon", "occurrence_id": "occurrence_meta_occurrence_meta_chameleon_winogrande", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_winogrande", "release_meta_derived_20240618_meta_chameleon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_meta_chameleon", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_apps_intro", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_apps_intro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_apps_intro", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_apps_intro", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_arc_challenge", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_arc_challenge", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bigpatent", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_bigpatent", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_bigpatent", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_bigpatent", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cnn_dailymail", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_cnn_dailymail", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_cnn_dailymail", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_cnn_dailymail", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codecontests", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_codecontests", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_codecontests", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_codecontests", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_copa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_copa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_copa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_copa", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_gsm8k", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_gsm8k", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_hellaswag", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_hellaswag", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_humaneval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_humaneval", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_mmlu", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_mbpp", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_mbpp", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_news", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_multi_news", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_multi_news", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_multi_news", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_natural_questions", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_natural_questions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_natural_questions", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_natural_questions", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_orangesum", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_orangesum", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_orangesum", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_orangesum", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_piqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_piqa", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_pn_summary", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_pn_summary", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_pn_summary", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_pn_summary", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_samsum", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_samsum", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_samsum", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_samsum", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_social_iqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_social_iqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_social_iqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_social_iqa", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_thaisum", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_thaisum", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_thaisum", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_thaisum", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_triviaqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_triviaqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_triviaqa", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wikisummary", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_wikisummary", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_wikisummary", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_wikisummary", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xsum", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20240618_multi_token_prediction_benchmark_xsum", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "model_id": "model_meta_model_meta_multi_token_prediction", "occurrence_id": "occurrence_meta_occurrence_meta_multi_token_prediction_xsum", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_xsum", "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "release_id": "release_meta_derived_20240618_multi_token_prediction", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_ai2d", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_ai2d", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_ai2d", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_bbh", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_big_bench_hard", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_big_bench_hard", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_chartqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_chartqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_chartqa", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_docvqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_docvqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_docvqa", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_drop", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_drop", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_drop", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_gpqa_diamond", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_gsm8k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_gsm8k", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_gsm8k", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_humaneval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_humaneval", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_humaneval", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_massive_multitask_language_understanding", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_massive_multitask_language_understanding", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_math", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_math", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_mathvista", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_mathvista", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_mathvista", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_mgsm", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_multilingual_grade_school_math", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_multilingual_grade_school_math", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_needle_in_a_haystack", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_needle_in_a_haystack", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_needle_in_a_haystack", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wildchat", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_wildchat", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_wildchat", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_wildchat", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xstest", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20240621_claude_3_5_sonnet_benchmark_xstest", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_xstest", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_xstest", "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240628_deepseek_chat_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "model_id": "model_deepseek_model_deepseek_v2_0628", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_0628_arena_hard", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_0628_arena_hard", "release_deepseek_20240628_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "release_id": "release_deepseek_20240628_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240628_deepseek_chat_benchmark_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "model_id": "model_deepseek_model_deepseek_v2_0628", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_0628_bbh", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_0628_bbh", "release_deepseek_20240628_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "release_id": "release_deepseek_20240628_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240628_deepseek_chat_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "model_id": "model_deepseek_model_deepseek_v2_0628", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_0628_humaneval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_0628_humaneval", "release_deepseek_20240628_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "release_id": "release_deepseek_20240628_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20240628_deepseek_chat_benchmark_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "model_id": "model_deepseek_model_deepseek_v2_0628", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20240628_deepseek_chat", "release_deepseek_20240628_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "release_id": "release_deepseek_20240628_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240628_deepseek_chat_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "model_id": "model_deepseek_model_deepseek_v2_0628", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_0628_math", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_0628_math", "release_deepseek_20240628_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "release_id": "release_deepseek_20240628_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_benchmark_humaneval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "occurrence_id": "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_humaneval", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_humaneval", "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_benchmark_lmarena", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "occurrence_id": "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_lmsys_chatbot_arena", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_lmsys_chatbot_arena", "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_benchmark_massive_multitask_language_understanding", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "occurrence_id": "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mmlu", "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_benchmark_mgsm", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "occurrence_id": "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mgsm", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mgsm", "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "occurrence_id": "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mmmu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mmmu", "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_adversarial_squad", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_adversarial_squad", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_adversarial_squad", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_adversarial_squad", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_agieval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_agieval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_agieval", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agieval_english", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_agieval_english", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_art_history", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_art_history", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_art_history", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_art_history", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_biology", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_biology", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_biology", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_biology", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_calculus", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_calculus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_calculus", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_calculus", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_chemistry", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_chemistry", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_chemistry", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_chemistry", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_english_language", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_english_language", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_english_language", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_english_language", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_english_literature", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_english_literature", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_english_literature", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_english_literature", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_environmental_science", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_environmental_science", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_environmental_science", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_environmental_science", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_macroeconomics", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_macroeconomics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_macroeconomics", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_macroeconomics", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_microeconomics", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_microeconomics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_microeconomics", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_microeconomics", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_physics", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_physics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_physics", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_physics", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_psychology", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_psychology", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_psychology", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_psychology", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_statistics", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_statistics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_statistics", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_statistics", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_united_states_government", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_united_states_government", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_us_government", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_us_government", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_united_states_history", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_united_states_history", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_us_history", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_us_history", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ap_world_history", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ap_world_history", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ap_world_history", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_world_history", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_api_bank", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_api_bank", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_api_bank", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_api_bank", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_arc_challenge", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_arc_challenge", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_bbh", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_big_bench_hard", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_big_bench_hard", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_berkeley_function_calling_leaderboard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_bfcl", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_bfcl", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_boolq", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_commonsenseqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_commonsenseqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_commonsenseqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_commonsenseqa", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cyberseceval_code_interpreter_abuse", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_cyberseceval_code_interpreter_abuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_code_interpreter_abuse", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_code_interpreter_abuse", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyberseceval_insecure_coding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_cyberseceval_insecure_coding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_insecure_coding", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_insecure_coding", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyberseceval_text_prompt_injection", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_cyberseceval_text_prompt_injection", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_text_prompt_injection", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_text_prompt_injection", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyberseceval_vulnerability_identification", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_cyberseceval_vulnerability_identification", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_vulnerability_identification", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_vulnerability_identification", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_drop", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_drop", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_drop", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_dynabench_squad", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_dynabench_squad", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_dynabench_squad", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_dynabench_squad", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gmat_quantitative", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_gmat_quantitative", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_gmat_quantitative", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gmat_quantitative", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gmat_verbal", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_gmat_verbal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_gmat_verbal", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gmat_verbal", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gorilla_api_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_gorilla_api_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_api_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_api_bench", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_gpqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_gpqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gpqa", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_main", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_gpqa_main", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gre_general_test_quantitative_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_gre_general_test_quantitative_reasoning", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_gre_quantitative", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gre_quantitative", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gre_general_test_verbal_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_gre_general_test_verbal_reasoning", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_gre_verbal", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gre_verbal", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gre_physics", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_gre_physics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_gre_physics", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gre_physics", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_gsm8k", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gsm8k", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm_plus", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_gsm_plus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_gsm_plus", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gsm_plus", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_humaneval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_humaneval", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_humaneval_plus", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_humaneval_plus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_humaneval_plus", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_humaneval_plus", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_ifeval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_ifeval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ifeval", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infinitebench_en_mc", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_infinitebench_en_mc", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_infinitebench_en_mc", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_infinitebench_en_mc", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infinitebench_en_qa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_infinitebench_en_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_infinitebench_en_qa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_infinitebench_en_qa", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_law_school_admission_test", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_law_school_admission_test", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_lsat", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_lsat", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_many_shot_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_many_shot_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_many_shot_gsm8k", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_many_shot_gsm8k", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_mmlu", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_math", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_math", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_mbpp", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_mbpp", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp_evalplus_base", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_mbpp_evalplus_base", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_mbpp_evalplus_base", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_mbpp_evalplus_base", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_meta_attack_automation_framework", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_meta_attack_automation_framework", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_meta_attack_automation", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_meta_attack_automation", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_meta_spear_phishing_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_meta_spear_phishing_benchmark", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_meta_spear_phishing", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_meta_spear_phishing", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_mgsm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_mgsm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_mgsm", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_mmlu_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_mmlu_pro", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_mmlu_pro", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_mmmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_multilingual_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_multilingual_mmlu", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_multipl_e", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_multiple", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_multiple", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nexus", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_nexus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_nexus", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_nexus", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nih_multi_needle", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_nih_multi_needle", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_nih_multi_needle", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_nih_multi_needle", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openbookqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_openbookqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_openbookqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_openbookqa", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_paws", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_paws", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_paws", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_paws", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_piqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_piqa", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qasper", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_qasper", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_qasper", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_qasper", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quac", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_quac", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_quac", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_quac", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_quality", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_quality", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_quality", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_quality", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_race", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_race", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_race", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_race", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_sat_math", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_sat_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_sat_math", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_sat_math", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_sat_reading", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_sat_reading", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_sat_reading", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_sat_reading", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_social_iqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_social_iqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_social_iqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_social_iqa", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_squad_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_squad_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_squad_v2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_squad_v2", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_squality", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_squality", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_squality", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_squality", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa_wiki", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_triviaqa_wiki", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_winogrande", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_winogrande", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_winogrande", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_worldsense", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat_benchmark_worldsense", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_1", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_1_worldsense", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_worldsense", "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_aime_2024", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_aime_2024", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_amc_2023", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_amc_2023", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_amc_2023", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_amc_2023", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_cmath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_cmath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_cmath", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cn_middle_school_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_cn_middle_school_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_cn_middle_school_2024", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_cn_middle_school_2024", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_collegemath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_collegemath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_collegemath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_collegemath", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gaokao", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_gaokao", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gaokao_math_cloze", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_gaokao_math_cloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao_math_cloze", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao_math_cloze", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gaokao_math_qa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_gaokao_math_qa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao_math_qa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao_math_qa", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gsm8k", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_math", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_stem", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_mmlu_stem", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_mmlu_stem", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_olympiadbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240807_introducing_qwen2_math_qwen2_math_benchmark_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_olympiadbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_olympiadbench", "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aitz", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_aitz", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_aitz", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_aitz", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alfred", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_alfred", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_alfred", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_alfred", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_chartqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_chartqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_chartqa", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_docvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_docvqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_docvqa", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_egoschema", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_egoschema", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_egoschema", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_fncall", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_fncall", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_fncall", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_fncall", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gym_cards", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_gym_cards", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_gym_cards", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_gym_cards", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hallbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_hallbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_hallbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_hallbench", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_infographicvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_infovqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_infovqa", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mathvision", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mathvista", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mathvista", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbench_1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_mmbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmbench_1_1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmbench_1_1", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mme", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_mme", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mme", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mme", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmmu", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmstar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmstar", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_mmt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmt_bench", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmvet", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_mmvet", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmvet", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmvet", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mtvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_mtvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mtvqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mtvqa", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mvbench", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocrbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_ocrbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_ocrbench", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_perception_test", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_perception_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_perceptiontest", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_perceptiontest", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_realworldqa", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_textvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_textvqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_textvqa", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_with_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_videomme_with_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_videomme_with_subtitles", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_without_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_benchmark_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_videomme_without_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_videomme_without_subtitles", "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_aider", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_aider", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_aider", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alignbench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_alignbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_alignbench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_alignbench", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_alpacaeval_2_0", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_alpacaeval_2_0", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_arena_hard", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_arena_hard", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ds_arena_code", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_ds_arena_code", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_ds_arena_code", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_ds_arena_code", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ds_fim_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_ds_fim_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_ds_fim_eval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_ds_fim_eval", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_humaneval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_humaneval", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_humaneval_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_humaneval_multilingual", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_humaneval_multilingual", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_livecodebench", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_mt_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_mt_bench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_mt_bench", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_swe_bench_verified", "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_year_not_specified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_aime_year_not_specified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_aime_year_not_specified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_aime_year_not_specified", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_bbq", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_bbq", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_bbq", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biolp_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_biolp_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_biolp_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_biolp_bench", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_challenging_refusal_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_challenging_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_challenging_refusal_evaluation", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_challenging_refusal_evaluation", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_changemyview", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_changemyview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_changemyview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_changemyview", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_codeforces", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_codeforces", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_collegiate_ctfs", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_collegiate_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_collegiate_ctfs", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_collegiate_ctfs", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_biology", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_gpqa_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_gpqa_biology", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_gpqa_biology", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_high_school_ctfs", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_high_school_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_high_school_ctfs", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_high_school_ctfs", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_human_sourced_jailbreaks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_human_sourced_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_human_sourced_jailbreaks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_human_sourced_jailbreaks", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_instruction_hierarchy_conflicts", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_instruction_hierarchy_conflicts", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_instruction_hierarchy_conflicts", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_instruction_hierarchy_conflicts", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_jailbreak_augmented_examples", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_jailbreak_augmented_examples", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_jailbreak_augmented_examples", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_jailbreak_augmented_examples", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_makemepay", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_makemepay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_makemepay", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_makemepay", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_makemesay", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_makemesay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_makemesay", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_makemesay", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mle_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_mle_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_mle_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_mle_bench", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_mmmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_mmmlu", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_research_engineer_interview", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_openai_research_engineer_interview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_openai_research_engineer_interview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_openai_research_engineer_interview", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_personqa", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_personqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_personqa", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_personqa", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_phrase_and_password_protection", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_phrase_and_password_protection", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_phrase_and_password_protection", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_phrase_and_password_protection", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_production_jailbreaks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_production_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_production_jailbreaks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_production_jailbreaks", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_professional_ctfs", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_professional_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_professional_ctfs", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_professional_ctfs", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_protocolqa_open_ended", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_protocolqa_open_ended", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_protocolqa_open_ended", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_protocolqa_open_ended", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quantbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_quantbench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_quantbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_quantbench", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_simpleqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_simpleqa", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_simpleqa", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_standard_refusal_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_standard_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_standard_refusal_evaluation", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_standard_refusal_evaluation", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_strongreject", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_strongreject", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_strongreject", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_strongreject", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_swe_bench_verified", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tutor_jailbreaks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_tutor_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_tutor_jailbreaks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_tutor_jailbreaks", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wildchat", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_wildchat", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wildchat", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wildchat", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmdp_biology", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_wmdp_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wmdp_biology", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wmdp_biology", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmdp_chemistry", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_wmdp_chemistry", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wmdp_chemistry", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wmdp_chemistry", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xstest", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_introducing_openai_o1_preview_benchmark_xstest", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_xstest", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_xstest", "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_year_not_specified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_aime_year_not_specified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_aime_year_not_specified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_aime_year_not_specified", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_bbq", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_bbq", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_bbq", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_biolp_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_biolp_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_biolp_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_biolp_bench", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_challenging_refusal_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_challenging_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_challenging_refusal_evaluation", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_challenging_refusal_evaluation", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_changemyview", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_changemyview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_changemyview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_changemyview", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_codeforces", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_codeforces", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_collegiate_ctfs", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_collegiate_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_collegiate_ctfs", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_collegiate_ctfs", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_gpqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_gpqa", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_gpqa", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_biology", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_gpqa_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_gpqa_biology", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_gpqa_biology", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_high_school_ctfs", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_high_school_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_high_school_ctfs", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_high_school_ctfs", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_human_sourced_jailbreaks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_human_sourced_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_human_sourced_jailbreaks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_human_sourced_jailbreaks", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_humaneval", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_humaneval", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_humaneval", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_instruction_hierarchy_conflicts", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_instruction_hierarchy_conflicts", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_instruction_hierarchy_conflicts", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_instruction_hierarchy_conflicts", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_jailbreak_augmented_examples", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_jailbreak_augmented_examples", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_jailbreak_augmented_examples", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_jailbreak_augmented_examples", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_makemepay", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_makemepay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_makemepay", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_makemepay", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_makemesay", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_makemesay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_makemesay", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_makemesay", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_massive_multitask_language_understanding", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mmlu", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_math", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_math", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_math", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mle_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_mle_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mle_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mle_bench", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mmmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mmmlu", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_openai_research_engineer_interview", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_openai_research_engineer_interview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_openai_research_engineer_interview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_openai_research_engineer_interview", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_personqa", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_personqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_personqa", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_personqa", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_phrase_and_password_protection", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_phrase_and_password_protection", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_phrase_and_password_protection", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_phrase_and_password_protection", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_production_jailbreaks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_production_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_production_jailbreaks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_production_jailbreaks", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_professional_ctfs", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_professional_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_professional_ctfs", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_professional_ctfs", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_protocolqa_open_ended", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_protocolqa_open_ended", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_protocolqa_open_ended", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_protocolqa_open_ended", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_quantbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_quantbench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_quantbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_quantbench", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_simpleqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_simpleqa", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_simpleqa", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_standard_refusal_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_standard_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_standard_refusal_evaluation", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_standard_refusal_evaluation", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_strongreject", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_strongreject", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_strongreject", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_strongreject", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_swe_bench_verified", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_tutor_jailbreaks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_tutor_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_tutor_jailbreaks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_tutor_jailbreaks", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_wildchat", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_wildchat", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wildchat", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wildchat", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_wmdp_biology", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_wmdp_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wmdp_biology", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wmdp_biology", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_wmdp_chemistry", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_wmdp_chemistry", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wmdp_chemistry", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wmdp_chemistry", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_xstest", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20240912_openai_o1_mini_benchmark_xstest", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_xstest", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_xstest", "release_openai_20240912_openai_o1_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_aider", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_aider", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alignbench_1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_alignbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_alignbench_1_1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_alignbench_1_1", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_arena_hard", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_arena_hard", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bigcodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_bigcodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_bigcodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_bigcodebench", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_bird_sql", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_bird_sql", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_bird_sql", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cruxeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_cruxeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_cruxeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_cruxeval", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_evalplus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_evalplus", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_gpqa", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_gsm8k", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_humaneval", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_ifeval", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_livebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_livebench", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_livecodebench", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_math", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mbpp", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mceval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_mceval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mceval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mceval", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mmlu_pro", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mmlu_redux", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mt_bench", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_multipl_e", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_multipl_e", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_spider", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_benchmark_spider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_spider", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_spider", "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_aider", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_aider", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_aime_2024", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_aime_2024", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_amc_2023", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_amc_2023", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_amc_2023", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_amc_2023", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bigcodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_bigcodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_bigcodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_bigcodebench", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_bird_sql", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_bird_sql", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_bird_sql", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_c_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_c_eval", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_collegemath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_collegemath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_collegemath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_collegemath", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cruxeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_cruxeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_cruxeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_cruxeval", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_evalplus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_evalplus", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gaokao_2023_english", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_gaokao_2023_english", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gaokao_2023_english", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gaokao_2023_english", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gpqa", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gsm8k", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_humaneval", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_ifeval", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_livecodebench", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mmlu", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mbpp", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mceval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_mceval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mceval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mceval", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mmlu_pro", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_multipl_e", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_multipl_e", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_olympiadbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_olympiadbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_olympiadbench", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_spider", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_benchmark_spider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_spider", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_spider", "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_alignbench_1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_alignbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_alignbench_1_1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_alignbench_1_1", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_ammlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_ammlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_ammlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_ammlu", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_arc_c", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_arc_c", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_arc_c", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_arena_hard", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_arena_hard", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_bbh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_bbh", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bigcodebench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_bigcodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_bird_sql", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_blend", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_blend", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_blend", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_blend", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cruxeval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_cruxeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_gpqa", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_gsm8k", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_hellaswag", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_hellaswag", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_hellaswag", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_humaneval", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_humaneval_plus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_humaneval_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_humaneval_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_humaneval_plus", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_ifeval", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_indommlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_indommlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_indommlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_indommlu", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_jmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_jmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_jmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_jmmlu", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_kmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_kmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_kmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_kmmlu", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_livebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_livebench", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_livecodebench", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_math", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mbpp", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp_plus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_mbpp_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mbpp_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mbpp_plus", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mceval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_mceval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mgsm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mgsm", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_mmlu_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_multilingual", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_pro", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_redux", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_stem", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_stem", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_stem", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mt_bench", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_multi_exam", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_multi_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_exam", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_exam", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_mathematics", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_multi_mathematics", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_mathematics", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_mathematics", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_translation", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_multi_translation", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_translation", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_translation", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_multi_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_understanding", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_understanding", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multipl_e", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multipl_e", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_okapi_mmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_okapi_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_okapi_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_okapi_mmlu", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_spider", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_spider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_theoremqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_theoremqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_theoremqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_theoremqa", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_truthfulqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_truthfulqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_truthfulqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_truthfulqa", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_turkishmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_turkishmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_turkishmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_turkishmmlu", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_benchmark_winogrande", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_winogrande", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_winogrande", "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_aime_2024", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_aime_2024", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_amc_2023", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_amc_2023", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_amc_2023", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_amc_2023", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_cmath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_cmath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_cmath", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cn_middle_school_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_cn_middle_school_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_cn_middle_school_2024", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_cn_middle_school_2024", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_collegemath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_collegemath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_collegemath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_collegemath", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gaokao", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_gaokao", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gaokao_math_cloze", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_gaokao_math_cloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao_math_cloze", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao_math_cloze", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gaokao_math_qa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_gaokao_math_qa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao_math_qa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao_math_qa", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gsm8k", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_math", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_minerva_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_minerva_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_minerva_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_minerva_math", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_stem", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_mmlu_stem", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_mmlu_stem", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_olympiadbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_benchmark_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "occurrence_id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_olympiadbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_olympiadbench", "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_hiddenmath", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_hiddenmath", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_math", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_math", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_mmlu_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_mmlu_pro", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval_english", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_agieval_english", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_agieval_english", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_agieval_english", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_ai2d", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_ai2d", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_ai2d", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_arc_challenge", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_arc_challenge", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_berkeley_function_calling_leaderboard_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_bfcl_v2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_bfcl_v2", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_chartqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_chartqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_chartqa", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_docvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_docvqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_docvqa", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_drop", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_drop", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_drop", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_gpqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_gpqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_gpqa", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_gsm8k", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_gsm8k", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_hellaswag", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_hellaswag", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_ifeval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_ifeval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_ifeval", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infinitebench_en_mc", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_infinitebench_en_mc", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_infinitebench_en_mc", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_infinitebench_en_mc", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infinitebench_en_qa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_infinitebench_en_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_infinitebench_en_qa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_infinitebench_en_qa", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infographicsvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_infographicsvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_infographicsvqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_infographicsvqa", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mmlu", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_math", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_math", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_mathvista", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_mathvista", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mathvista", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_mgsm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_mgsm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mgsm", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_mmmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_multilingual_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_multilingual_mmlu", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_mmmu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_mmmu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mmmu", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro_standard", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_mmmu_pro_standard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_mmmu_pro_standard", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mmmu_pro_standard", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro_vision", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_mmmu_pro_vision", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_mmmu_pro_vision", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mmmu_pro_vision", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_needle_in_a_haystack", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_needle_in_haystack", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_needle_in_haystack", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nexus", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_nexus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_nexus", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_nexus", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nih_multi_needle", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_nih_multi_needle", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_nih_multi_needle", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_nih_multi_needle", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_open_rewrite", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_open_rewrite", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_open_rewrite", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_open_rewrite", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quac", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_quac", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quac", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quac", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_squad_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_squad_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_squad_v2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_squad_v2", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_textvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_textvqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_textvqa", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tldr9", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_tldr9", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_tldr9_plus", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_tldr9_plus", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op_benchmark_vqav2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_3_2", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_vqav2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_vqav2", "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_code_llama_checkpoints_benchmark_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_code", "model_id": "model_meta_model_meta_layerskip_code_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_code_llama_humaneval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_code_llama_humaneval", "release_meta_derived_20241018_code_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_code_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_code_llama_checkpoints_benchmark_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_code", "model_id": "model_meta_model_meta_layerskip_code_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_code_llama_mbpp", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_code_llama_mbpp", "release_meta_derived_20241018_code_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_code_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_arc_challenge", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_arc_challenge", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_arc_easy", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_arc_easy", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_arc_easy", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_boolq", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_boolq", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_boolq", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_commonsenseqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_commonsenseqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_commonsenseqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_commonsenseqa", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_gsm8k", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_gsm8k", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_hellaswag", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_hellaswag", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_humaneval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_humaneval", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_mmlu", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_math", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_math", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_mbpp", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_mbpp", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_natural_questions", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_natural_questions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_natural_questions", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_natural_questions", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openbookqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_openbookqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_openbookqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_openbookqa", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_piqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_piqa", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_race_high", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_race_high", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_race_high", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_race_high", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_race_middle", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_race_middle", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_race_middle", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_race_middle", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_triviaqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_triviaqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_triviaqa", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241018_layer_skip_llama_checkpoints_benchmark_winogrande", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "model_id": "model_meta_model_meta_layerskip_llama", "occurrence_id": "occurrence_meta_occurrence_meta_layerskip_llama_winogrande", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_winogrande", "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_ai2d", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_ai2d", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_ai2d", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_aime_2024", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_american_invitational_mathematics_examination_2024", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_american_invitational_mathematics_examination_2024", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_bbh", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_big_bench_hard", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_big_bench_hard", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_bench", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_chartqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_chartqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_chartqa", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_docvqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_docvqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_docvqa", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_drop", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_drop", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_drop", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_gpqa_diamond", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_humaneval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_humaneval", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_humaneval", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_ifeval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_instruction_following_evaluation", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_instruction_following_evaluation", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_massive_multitask_language_understanding", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_massive_multitask_language_understanding", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_math", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_math", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mathvista", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mathvista", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mathvista", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mgsm", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_multilingual_grade_school_math", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_multilingual_grade_school_math", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mmlu_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mmlu_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mmlu_pro", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_osworld", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_osworld", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_swe_bench_verified", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wildchat", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_wildchat", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_wildchat", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_wildchat", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xstest", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_xstest", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_xstest", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_xstest", "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_arc_challenge", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_arc_challenge", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_berkeley_function_calling_leaderboard_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_bfcl_v2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_bfcl_v2", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_gpqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_gpqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_gpqa", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_gsm8k", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_gsm8k", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_hellaswag", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_hellaswag", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_ifeval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_ifeval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_ifeval", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_mmlu", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_math", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_math", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_mgsm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_mgsm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_mgsm", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nexus", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_nexus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_nexus", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_nexus", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_open_rewrite", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_open_rewrite", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_open_rewrite", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_open_rewrite", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tldr9", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20241024_introducing_quantized_llama_models_with_increased_sp_benchmark_tldr9", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_2_quantized", "occurrence_id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_tldr9_plus", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_tldr9_plus", "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_aider", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_aider", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2023", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_amc_2023", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bigcodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_bigcodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_bigcodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_bigcodebench", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_bird_sql", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_code_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_code_arena", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_code_arena", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_code_arena", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_collegemath", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_collegemath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_crosscodeeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_crosscodeeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_crosscodeeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_crosscodeeval", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_crosscodelongeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_crosscodelongeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_crosscodelongeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_crosscodelongeval", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cruxeval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_cruxeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_evalplus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_evalplus", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gaokao_2023_english", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_gaokao_2023_english", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval_infilling", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_humaneval_infilling", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_humaneval_infilling", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_humaneval_infilling", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_livecodebench", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mbpp", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mceval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_mceval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mceval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mceval", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mdeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_mdeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mdeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mdeval", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_olympiadbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_repoeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_repoeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_repoeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_repoeval", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_safim", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_safim", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_safim", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_safim", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_spider", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_benchmark_spider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_alignbench_1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_alignbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_alignbench_1_1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_alignbench_1_1", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_ammlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_ammlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_arc_c", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_arena_hard", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_arena_hard", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bigcodebench", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_bigcodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_bird_sql", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blend", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_blend", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cruxeval", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_cruxeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_gpqa", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_gsm8k", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_hellaswag", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_humaneval", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_humaneval_plus", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_humaneval_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_ifeval", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_indommlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_indommlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_jmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_jmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_kmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_kmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_livebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_livebench", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_longbench_chat", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_longbench_chat", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_longbench_chat", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_longbench_chat", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lv_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_lv_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_lv_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_lv_eval", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_math", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mbpp", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp_plus", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_mbpp_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mceval", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_mceval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_multilingual", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_mmlu_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mmlu_pro", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mmlu_redux", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_stem", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_exam", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_multi_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_mathematics", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_multi_mathematics", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_translation", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_multi_translation", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_multi_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_okapi_mmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_okapi_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_passkey_retrieval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_passkey_retrieval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_passkey_retrieval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_passkey_retrieval", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ruler", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_ruler", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_ruler", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_ruler", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_spider", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_spider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_theoremqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_theoremqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_truthfulqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_truthfulqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_turkishmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_turkishmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_benchmark_winogrande", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241120_deepseek_r1_lite_preview_benchmark_aime", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_lite_preview", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_lite_preview_aime", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_lite_preview_aime", "release_deepseek_20241120_deepseek_r1_lite_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-20", "release_id": "release_deepseek_20241120_deepseek_r1_lite_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241120_deepseek_r1_lite_preview_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_lite_preview", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_lite_preview_math", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_lite_preview_math", "release_deepseek_20241120_deepseek_r1_lite_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-20", "release_id": "release_deepseek_20241120_deepseek_r1_lite_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_aime_2024", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_aime_2024", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_amc_2023", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_amc_2023", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_cmath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cn_middle_school_2024", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_cn_middle_school_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_collegemath", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_collegemath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gaokao", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_gaokao", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gaokao_math_cloze", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_gaokao_math_cloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gaokao_math_qa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_gaokao_math_qa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_gpqa", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_livecodebench", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_math_500", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_math_500", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_math_500", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_minerva_math", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_minerva_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_stem", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_olympiadbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_benchmark_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_aider", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_alignbench", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_alignbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ds_arena_code", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_ds_arena_code", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ds_fim_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_ds_fim_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval_multilingual", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_humaneval_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_1210_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_1210_livecodebench", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v2_5_1210_math_500", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_1210_math_500", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_mt_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20241210_deepseek_chat_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20241210_deepseek_chat", "release_deepseek_20241210_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_arc_challenge", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_arc_challenge", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_arc_easy", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_arc_easy", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_arc_easy", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cute", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_cute", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_cute", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_cute", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_flores_101", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_flores_101", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_flores_101", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_flores_101", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_hellaswag", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_hellaswag", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag_noise", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_hellaswag_noise", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_hellaswag_noise", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_hellaswag_noise", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_humaneval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_humaneval", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_mmlu", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_mbpp", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_mbpp", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_phonology_bench_g2p", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_phonology_bench_g2p", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_phonology_bench_g2p", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_phonology_bench_g2p", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_dynamic_byte_latent_transformer_benchmark_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "model_id": "model_meta_model_meta_dynamic_blt", "occurrence_id": "occurrence_meta_occurrence_meta_dynamic_blt_piqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_piqa", "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c4", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_c4", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_c4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_c4", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cnn_dailymail", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_cnn_dailymail", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_cnn_dailymail", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_cnn_dailymail", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cosmopedia_coherence", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_cosmopedia_coherence", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_cosmopedia_coherence", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_cosmopedia_coherence", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_flores", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_flores", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_flores", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_flores", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lcfo_long_summary_condition", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_lcfo_long_summary_condition", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_lcfo_twenty", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_lcfo_twenty", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lcfo_medium_summary_condition", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_lcfo_medium_summary_condition", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_lcfo_ten", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_lcfo_ten", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lcfo_short_summary_condition", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_lcfo_short_summary_condition", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_lcfo_five", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_lcfo_five", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_project_gutenberg", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_project_gutenberg", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_gutenberg", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_gutenberg", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_rocstories", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_rocstories", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_rocstories", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_rocstories", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wikipedia_english", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_wikipedia_english", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_wikipedia_en", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_wikipedia_en", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xsum", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20241212_meta_large_concept_models_benchmark_xsum", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "model_id": "model_meta_model_meta_large_concept_models", "occurrence_id": "occurrence_meta_occurrence_meta_large_concept_models_xsum", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_xsum", "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_aime_2024", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_aime_2024", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_aime_2024", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_bbq", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_bbq", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_bbq", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biolp_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_biolp_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_biolp_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_biolp_bench", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_challenging_refusal_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_challenging_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_challenging_refusal_evaluation", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_challenging_refusal_evaluation", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_changemyview", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_changemyview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_changemyview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_changemyview", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_collegiate_ctfs", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_collegiate_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_collegiate_ctfs", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_collegiate_ctfs", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_biology", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_gpqa_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_gpqa_biology", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_gpqa_biology", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_gpqa_diamond", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_high_school_ctfs", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_high_school_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_high_school_ctfs", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_high_school_ctfs", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_human_sourced_jailbreaks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_human_sourced_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_human_sourced_jailbreaks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_human_sourced_jailbreaks", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_instruction_hierarchy_conflicts", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_instruction_hierarchy_conflicts", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_instruction_hierarchy_conflicts", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_instruction_hierarchy_conflicts", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_jailbreak_augmented_examples", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_jailbreak_augmented_examples", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_jailbreak_augmented_examples", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_jailbreak_augmented_examples", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livebench_coding", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_livebench_coding", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_livebench_coding", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_livebench_coding", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_makemepay", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_makemepay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_makemepay", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_makemepay", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_makemesay", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_makemesay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_makemesay", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_makemesay", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_massive_multitask_language_understanding", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmlu", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_math", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_math", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_math", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_mathvista", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mathvista", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mathvista", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_mgsm", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mgsm", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mgsm", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mle_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_mle_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mle_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mle_bench", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmmlu", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmmu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmmu", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_research_engineer_interview", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_openai_research_engineer_interview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_openai_research_engineer_interview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_openai_research_engineer_interview", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_personqa", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_personqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_personqa", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_personqa", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_phrase_and_password_protection", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_phrase_and_password_protection", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_phrase_and_password_protection", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_phrase_and_password_protection", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_production_jailbreaks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_production_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_production_jailbreaks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_production_jailbreaks", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_professional_ctfs", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_professional_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_professional_ctfs", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_professional_ctfs", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_protocolqa_open_ended", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_protocolqa_open_ended", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_protocolqa_open_ended", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_protocolqa_open_ended", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quantbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_quantbench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_quantbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_quantbench", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_simpleqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_simpleqa", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_simpleqa", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_standard_refusal_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_standard_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_standard_refusal_evaluation", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_standard_refusal_evaluation", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_strongreject", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_strongreject", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_strongreject", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_strongreject", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_swe_bench_verified", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau_bench_airline", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_tau_bench_airline", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tau_bench_airline", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tau_bench_airline", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau_bench_retail", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_tau_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tau_bench_retail", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tau_bench_retail", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tutor_jailbreaks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_tutor_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tutor_jailbreaks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tutor_jailbreaks", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wildchat", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_wildchat", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wildchat", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wildchat", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmdp_biology", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_wmdp_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wmdp_biology", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wmdp_biology", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmdp_chemistry", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_wmdp_chemistry", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wmdp_chemistry", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wmdp_chemistry", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xstest", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20241217_openai_o1_and_new_tools_for_developers_benchmark_xstest", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "occurrence_id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_xstest", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_xstest", "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_adversarial_squad", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_adversarial_squad", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_agieval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval_english", "consecutive_reviewed_omissions": 2, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_agieval_english", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_art_history", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_art_history", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_biology", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_biology", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_calculus", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_calculus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_chemistry", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_chemistry", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_english_language", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_english_language", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_english_literature", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_english_literature", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_environmental_science", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_environmental_science", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_macroeconomics", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_macroeconomics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_microeconomics", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_microeconomics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_physics", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_physics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_psychology", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_psychology", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_statistics", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_statistics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_united_states_government", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_united_states_government", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_united_states_history", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_united_states_history", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ap_world_history", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ap_world_history", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_api_bank", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_api_bank", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_bbh", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_berkeley_function_calling_leaderboard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 2, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_boolq", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_commonsenseqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_commonsenseqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cyberseceval_code_interpreter_abuse", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_cyberseceval_code_interpreter_abuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cyberseceval_insecure_coding", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_cyberseceval_insecure_coding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cyberseceval_text_prompt_injection", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_cyberseceval_text_prompt_injection", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cyberseceval_vulnerability_identification", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_cyberseceval_vulnerability_identification", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_drop", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_dynabench_squad", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_dynabench_squad", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gmat_quantitative", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_gmat_quantitative", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gmat_verbal", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_gmat_verbal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gorilla_api_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_gorilla_api_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_gpqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_main", "consecutive_reviewed_omissions": 2, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_gpqa_main", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gre_general_test_quantitative_reasoning", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_gre_general_test_quantitative_reasoning", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gre_general_test_verbal_reasoning", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_gre_general_test_verbal_reasoning", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gre_physics", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_gre_physics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm_plus", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_gsm_plus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval_plus", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_humaneval_plus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_ifeval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_infinitebench_en_mc", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_infinitebench_en_mc", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_infinitebench_en_qa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_infinitebench_en_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_law_school_admission_test", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_law_school_admission_test", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_many_shot_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_many_shot_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp_evalplus_base", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_mbpp_evalplus_base", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_meta_attack_automation_framework", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_meta_attack_automation_framework", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_meta_spear_phishing_benchmark", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_meta_spear_phishing_benchmark", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_mgsm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_mmlu_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_mmmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_multipl_e", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_nexus", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_nexus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_nih_multi_needle", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_nih_multi_needle", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openbookqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_openbookqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_paws", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_paws", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_qasper", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_qasper", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_quac", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_quac", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_quality", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_quality", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_race", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_race", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_sat_math", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_sat_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_sat_reading", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_sat_reading", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_social_iqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_social_iqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_squad_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_squad_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_squality", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_squality", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_triviaqa_wiki", "consecutive_reviewed_omissions": 2, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_triviaqa_wiki", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_winogrande", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_worldsense", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20241219_the_future_of_ai_built_with_llama_benchmark_worldsense", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "model_id": "model_meta_model_meta_llama_3_3_70b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20241219_the_future_of_ai_built_with_llama", "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "release_id": "release_meta_20241219_the_future_of_ai_built_with_llama", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "model_id": "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mathvision", "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "release_id": "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_benchmark_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "model_id": "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mathvista", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mathvista", "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "release_id": "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "model_id": "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mmmu", "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "release_id": "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_olympiadbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_benchmark_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "model_id": "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_olympiadbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_olympiadbench", "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "release_id": "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_agieval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_agieval", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_edit", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_aider_edit", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_aider_edit", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_aider_edit", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_aider_polyglot", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_aider_polyglot", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_aime_2024", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_aime_2024", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_alpacaeval_2_0", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_alpacaeval_2_0", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_arc_c", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_arc_challenge", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_arc_challenge", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_arc_easy", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_arc_easy", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_arc_easy", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_arena_hard", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_arena_hard", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_bbh", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_bbh", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c3", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_c3", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_c3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_c3", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_c_eval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_c_eval", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_c_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_c_simpleqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_c_simpleqa", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ccpm", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_ccpm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_ccpm", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_ccpm", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_cluewsc", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cluewsc", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_cmath", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cmath", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_cmmlu", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cmmlu", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmrc", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_cmrc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_cmrc", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cmrc", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cnmo_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_cnmo_2024", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cnmo_2024", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_codeforces", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_codeforces", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cruxeval_i", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_cruxeval_i", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_cruxeval_i", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cruxeval_i", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cruxeval_o", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_cruxeval_o", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_cruxeval_o", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cruxeval_o", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_drop", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_drop", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frames", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_frames", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_frames", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_gpqa_diamond", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_gsm8k", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_gsm8k", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_hellaswag", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_hellaswag", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_humaneval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_humaneval", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_humaneval_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_humaneval_multilingual", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_humaneval_multilingual", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_ifeval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_ifeval", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_livecodebench", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_base", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_livecodebench_base", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_livecodebench_base", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_livecodebench_base", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_longbench_v2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_longbench_v2", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_mmlu", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mmlu", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_math", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_math", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_math_500", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_math_500", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_mbpp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_mbpp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mbpp", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_mgsm", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mgsm", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_mmlu_pro", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mmlu_pro", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_mmlu_redux", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mmlu_redux", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu_non_english", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_mmmlu_non_english", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_mmmlu_non_english", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mmmlu_non_english", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_naturalquestions", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_naturalquestions", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_naturalquestions", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_naturalquestions", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_needle_in_a_haystack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_needle_in_a_haystack", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_needle_in_a_haystack", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_pile_test", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_pile_test", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_pile_test", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_pile_test", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_piqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_piqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_piqa", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_race_high", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_race_high", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_race_high", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_race_high", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_race_middle", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_race_middle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_race_middle", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_race_middle", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_simpleqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_simpleqa", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_swe_bench_verified", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_triviaqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_triviaqa", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20241226_deepseek_chat_benchmark_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_winogrande", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_winogrande", "release_deepseek_20241226_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_aider_polyglot", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_aider_polyglot", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_aime", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250120_deepseek_reasoner", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_aime_2024", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_aime_2024", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_alpacaeval_2_0", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_alpacaeval_2_0", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_arena_hard", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_arena_hard", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_c_eval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_c_eval", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_c_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_c_simpleqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_c_simpleqa", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_cluewsc", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_cluewsc", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cnmo_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_cnmo_2024", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_cnmo_2024", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_codeforces", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_codeforces", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_drop", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_drop", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frames", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_frames", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_frames", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_gpqa_diamond", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_ifeval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_ifeval", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_livecodebench", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_mmlu", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_mmlu", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250120_deepseek_reasoner", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_math_500", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_math_500", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_mmlu_pro", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_mmlu_pro", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_mmlu_redux", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_mmlu_redux", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_simpleqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_simpleqa", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250120_deepseek_reasoner_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_swe_bench_verified", "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250123_computer_using_agent_benchmark_osworld", "lab_id": "openai", "lineage_id": "lineage_openai_computer_using_agent", "model_id": "model_openai_model_openai_20250123_computer_using_agent", "occurrence_id": "occurrence_openai_occurrence_openai_20250123_computer_using_agent_osworld", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250123_computer_using_agent_osworld", "release_openai_20250123_computer_using_agent" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "release_id": "release_openai_20250123_computer_using_agent", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_webarena", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250123_computer_using_agent_benchmark_webarena", "lab_id": "openai", "lineage_id": "lineage_openai_computer_using_agent", "model_id": "model_openai_model_openai_20250123_computer_using_agent", "occurrence_id": "occurrence_openai_occurrence_openai_20250123_computer_using_agent_webarena", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250123_computer_using_agent_webarena", "release_openai_20250123_computer_using_agent" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "release_id": "release_openai_20250123_computer_using_agent", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_webvoyager", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250123_computer_using_agent_benchmark_webvoyager", "lab_id": "openai", "lineage_id": "lineage_openai_computer_using_agent", "model_id": "model_openai_model_openai_20250123_computer_using_agent", "occurrence_id": "occurrence_openai_occurrence_openai_20250123_computer_using_agent_webvoyager", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250123_computer_using_agent_webvoyager", "release_openai_20250123_computer_using_agent" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "release_id": "release_openai_20250123_computer_using_agent", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_alignbench_1_1", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_alignbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ammlu", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_ammlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_arc_c", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_arena_hard", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_arena_hard", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bigcodebench", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_bigcodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_bird_sql", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blend", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_blend", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cruxeval", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_cruxeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_gpqa", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_gsm8k", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_hellaswag", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_humaneval", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_humaneval_plus", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_humaneval_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_ifeval", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_indommlu", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_indommlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_jmmlu", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_jmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_kmmlu", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_kmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_livebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_livebench", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_livecodebench", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_longbench_chat", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_longbench_chat", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_longbench_chat", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_longbench_chat", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_lv_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_lv_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_lv_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_lv_eval", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_math", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mbpp", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp_plus", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_mbpp_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mceval", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_mceval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_multilingual", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_mmlu_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mmlu_pro", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mmlu_redux", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_stem", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mt_bench", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_multi_exam", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_multi_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_mathematics", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_multi_mathematics", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_translation", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_multi_translation", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_understanding", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_multi_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_multipl_e", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_multipl_e", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_okapi_mmlu", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_okapi_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_passkey_retrieval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_passkey_retrieval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_passkey_retrieval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_passkey_retrieval", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_ruler", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_ruler", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_ruler", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_ruler", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_spider", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_spider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_theoremqa", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_theoremqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_truthfulqa", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_truthfulqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_turkishmmlu", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_turkishmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_benchmark_winogrande", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aitz", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_aitz", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_aitz", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_aitz", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_android_control", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_android_control", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_android_control", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_android_control", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_androidworld", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_androidworld", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_cc_ocr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_cc_ocr", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charadessta", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_charadessta", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_charadessta", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_charadessta", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_docvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_docvqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_docvqa", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_infographicvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_infovqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_infovqa", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_lvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_lvbench", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mathvision", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mathvista", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mathvista", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_megabench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_megabench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_megabench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_megabench", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbench_1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_mmbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmbench_1_1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmbench_1_1", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbench_video", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_mmbench_video", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmbench_video", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmbench_video", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmmu", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmmu_pro", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmstar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmstar", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocrbenchv2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_ocrbenchv2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_ocrbenchv2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_ocrbenchv2", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_osworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_osworld", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_osworld", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_screenspot", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_screenspot", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_screenspot", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_screenspot_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_screenspot_pro", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_benchmark_videomme", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_videomme", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_videomme", "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 4, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_alignbench_1_1", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_alignbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ammlu", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_ammlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_arc_c", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_arena_hard", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_arena_hard", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bigcodebench", "consecutive_reviewed_omissions": 4, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_bigcodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 4, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_bird_sql", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blend", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_blend", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cruxeval", "consecutive_reviewed_omissions": 4, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_cruxeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 4, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_gpqa_diamond", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_hellaswag", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval_plus", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_humaneval_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_indommlu", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_indommlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_jmmlu", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_jmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_kmmlu", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_kmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_livebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_livebench", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_livecodebench", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_longbench_chat", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_longbench_chat", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lv_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_lv_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp_plus", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_mbpp_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mceval", "consecutive_reviewed_omissions": 4, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_mceval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_multilingual", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_mmlu_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_mmlu_pro", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_stem", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mt_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_exam", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_multi_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_mathematics", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_multi_mathematics", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_translation", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_multi_translation", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_understanding", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_multi_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_okapi_mmlu", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_okapi_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_passkey_retrieval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_passkey_retrieval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ruler", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_ruler", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_spider", "consecutive_reviewed_omissions": 4, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_spider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_theoremqa", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_theoremqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_truthfulqa", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_truthfulqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_turkishmmlu", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_turkishmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_benchmark_winogrande", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250131_openai_o3_mini_benchmark_aime_2024", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_aime_2024", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_aime_2024", "release_openai_20250131_openai_o3_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250131_openai_o3_mini_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_codeforces", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_codeforces", "release_openai_20250131_openai_o3_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontiermath", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250131_openai_o3_mini_benchmark_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_frontiermath", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_frontiermath", "release_openai_20250131_openai_o3_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250131_openai_o3_mini_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_gpqa_diamond", "release_openai_20250131_openai_o3_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livebench_coding", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250131_openai_o3_mini_benchmark_livebench_coding", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_livebench_coding", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_livebench_coding", "release_openai_20250131_openai_o3_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250131_openai_o3_mini_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_swe_bench_verified", "release_openai_20250131_openai_o3_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "release_id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bias_benchmark_for_question_answering", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bias_benchmark_for_question_answering", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bench", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bioweapons_knowledge_questions", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bioweapons_knowledge_questions", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bioweapons_knowledge_questions", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bioweapons_knowledge_questions", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_cybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_cybench", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyber_harness_network_challenges", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_cyber_harness_network_challenges", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_cyber_harness_network_challenges", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_cyber_harness_network_challenges", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_gpqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_gpqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_gpqa", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_internal_ai_research_evaluation_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_internal_ai_research_evaluation_suite", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_internal_ai_research_evaluation_suite", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_labbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_labbench_cloningscenarios", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_cloningscenarios", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_cloningscenarios", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_labbench_figqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_figqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_figqa", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_labbench_protocolqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_protocolqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_protocolqa", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_labbench_seqqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_seqqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_seqqa", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_long_form_virology_tasks", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_long_form_virology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_long_form_virology_tasks", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_long_form_virology_tasks", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_massive_multitask_language_understanding", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_massive_multitask_language_understanding", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_metr_data_deduplication", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_metr_data_deduplication", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_metr_data_deduplication", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_metr_data_deduplication", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_re_bench_subset", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_re_bench_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_re_bench_subset", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_re_bench_subset", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_secure_bio_lab_assistance_benchmark_multimodal", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_secure_bio_lab_assistance_benchmark_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_secure_bio_lab_assistance_benchmark_multimodal", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_secure_bio_lab_assistance_benchmark_multimodal", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_swe_bench_verified", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_swe_bench_verified_hard_subset", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_swe_bench_verified_hard_subset", "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2023", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_amc_2023", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_cmath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cn_middle_school_2024", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_cn_middle_school_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_collegemath", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_collegemath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gaokao", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_gaokao", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gaokao_math_cloze", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_gaokao_math_cloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gaokao_math_qa", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_gaokao_math_qa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_math_500", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_minerva_math", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_minerva_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_stem", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_olympiadbench", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview_benchmark_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "release_id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250227_introducing_gpt_4_5_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_aime_2025", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_aime_2025", "release_openai_20250227_introducing_gpt_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250227_introducing_gpt_4_5_benchmark_gpqa", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_gpqa", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_gpqa", "release_openai_20250227_introducing_gpt_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250227_introducing_gpt_4_5_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_mmmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_mmmlu", "release_openai_20250227_introducing_gpt_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250227_introducing_gpt_4_5_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_mmmu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_mmmu", "release_openai_20250227_introducing_gpt_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250227_introducing_gpt_4_5_benchmark_simpleqa", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_simpleqa", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_simpleqa", "release_openai_20250227_introducing_gpt_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250227_introducing_gpt_4_5_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_swe_bench_verified", "release_openai_20250227_introducing_gpt_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_lancer_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250227_introducing_gpt_4_5_benchmark_swe_lancer_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_swe_lancer_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_swe_lancer_diamond", "release_openai_20250227_introducing_gpt_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_aime_2024", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_aime_2024", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_amc_2023", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_amc_2023", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_berkeley_function_calling_leaderboard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_bfcl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_bfcl", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_cmath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cn_middle_school_2024", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_cn_middle_school_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_collegemath", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_collegemath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gaokao", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_gaokao", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gaokao_math_cloze", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_gaokao_math_cloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gaokao_math_qa", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_gaokao_math_qa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_ifeval", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_livebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_livebench", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_livecodebench", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_math_500", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_minerva_math", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_minerva_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_stem", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_olympiadbench", "consecutive_reviewed_omissions": 3, "id": "status_release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_benchmark_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_activitynet_qa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_activitynet_qa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_agieval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_agieval", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_agieval", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_arc_c", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_arc_challenge", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_arc_challenge", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_arc_easy", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_arc_easy", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bigbench_hard", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bigbench_hard", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_big_bench_extra_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_big_bench_extra_hard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bigbench_extra_hard", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bigbench_extra_hard", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_bird_sql", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bird_sql", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bird_sql", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_blink", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_blink", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_boolq", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_boolq", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_chartqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_chartqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_coco_caption", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_coco_caption", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_coco_caption", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_coco_caption", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_countbench_vqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_countbench_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_countbench_vqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_countbench_vqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_docvqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_docvqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_drop", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_drop", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_eclektic", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_eclektic", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_eclektic", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_eclektic", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_facts_grounding", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_facts_grounding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_facts_grounding", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_facts_grounding", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_flores", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_flores", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_flores", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_flores", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_global_mmlu_lite", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_global_mmlu_lite", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_global_mmlu_lite", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_global_mmlu_lite", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_gpqa_diamond", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_gsm8k", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_gsm8k", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_hellaswag", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_hellaswag", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_hiddenmath", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_hiddenmath", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_human_eval", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_human_eval", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_infographicvqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_infographicvqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_livecodebench", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_livecodebench", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_lmarena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_lmarena", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmlu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmlu", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_math", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_math", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mathvista", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mathvista", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_mbpp", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mbpp", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mbpp", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mgsm", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mgsm", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmlu_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmlu_pro", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmmu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmmu", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_natural2code", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_natural2code", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_natural_questions", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_natural_questions", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_natural_questions", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_natural_questions", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ok_vqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_ok_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_ok_vqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_ok_vqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_perception_test_mcvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_perception_test_mcvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_perception_test_mcvqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_perception_test_mcvqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_piqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_piqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_realworldqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_realworldqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_simpleqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_simpleqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_simpleqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_socialiqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_socialiqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_social_iqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_social_iqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_spatialsense_vqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_spatialsense_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_spatialsense_vqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_spatialsense_vqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tallyqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_tallyqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_tallyqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_tallyqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_textvqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_textvqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_triviaqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_triviaqa", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_vqav2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_vqav2", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_winogrande", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_winogrande", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_winogrande", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_wmt24", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_wmt24_plus_plus", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_wmt24_plus_plus", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_xquad", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250312_introducing_gemma_3_benchmark_xquad", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_xquad", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_xquad", "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aitz", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_aitz", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_android_control", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_android_control", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_charadessta", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_charadessta", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_docvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_docvqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_docvqa", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_gpqa_diamond", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_humaneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_humaneval", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_infographicvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_math", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mathvista", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mathvista", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mbpp", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_megabench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_megabench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mm_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_mm_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mm_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mm_mt_bench", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbench_1_1", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_mmbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmbench_video", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_mmbench_video", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmlu_pro", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmmu", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmmu_pro", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ocrbenchv2", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_ocrbenchv2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_osworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_screenspot", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_screenspot", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_benchmark_videomme", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_edit", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_aider_edit", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_aime", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_0324_aime", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_0324_aime", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_arc_c", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_arc_easy", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_c3", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_c3", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_simpleqa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_c_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ccpm", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_ccpm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmrc", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_cmrc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cnmo_2024", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cruxeval_i", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_cruxeval_i", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cruxeval_o", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_cruxeval_o", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_frames", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_gpqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_0324_gpqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_0324_gpqa", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval_multilingual", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_humaneval_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_0324_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_0324_livecodebench", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench_base", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_livecodebench_base", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_mbpp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_0324_mmlu_pro", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_0324_mmlu_pro", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu_non_english", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_mmmlu_non_english", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_naturalquestions", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_naturalquestions", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_needle_in_a_haystack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_pile_test", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_pile_test", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_piqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_race_high", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_race_high", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_race_middle", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_race_middle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250324_deepseek_chat_benchmark_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_0324", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250324_deepseek_chat", "release_deepseek_20250324_deepseek_chat" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_aime_2025", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_aime_2025", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_gpqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_gpqa", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_humanitys_last_exam", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_lmarena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_lmarena", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_mmlu_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_mmlu_pro", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_swe_bench_verified", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aitz", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_aitz", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_android_control", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_android_control", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_charadessta", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_charadessta", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_common_voice", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_common_voice", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_common_voice", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_common_voice", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_covost2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_covost2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_covost2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_covost2", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_docvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_gsm8k", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_infographicvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmlu", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_megabench", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_megabench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mm_mt_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mm_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmau", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mmau", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmau", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmau", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbench_1_1", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mmbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmbench_video", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mmbench_video", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmmu", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmstar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmstar", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_mvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mvbench", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocrbenchv2", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_ocrbenchv2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_omnibench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_omnibench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_omnibench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_omnibench", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_osworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_screenspot", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_screenspot", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_seed_tts_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_seed_tts_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_seed_tts_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_seed_tts_eval", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_benchmark_videomme", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "model_id": "model_qwen_model_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview_mathvision", "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-27", "release_id": "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview_benchmark_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "model_id": "model_qwen_model_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-27", "release_id": "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "model_id": "model_qwen_model_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-27", "release_id": "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_olympiadbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview_benchmark_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "model_id": "model_qwen_model_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-27", "release_id": "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval_english", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_agieval_english", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_ai2d", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_berkeley_function_calling_leaderboard_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_chartqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_chartqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_chartqa", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_docvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_docvqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_docvqa", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_drop", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_gpqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_gpqa_diamond", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_gpqa_diamond", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_ifeval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_infinitebench_en_mc", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_infinitebench_en_mc", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_infinitebench_en_qa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_infinitebench_en_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicsvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_infographicsvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_livecodebench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_livecodebench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_livecodebench", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_mmlu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mmlu", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_math", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_math", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mathvista", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_mathvista", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mathvista", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_mbpp", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mbpp", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mgsm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_mgsm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mgsm", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mmlu_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_mmlu_pro", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mmlu_pro", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mmmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mmmu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_mmmu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mmmu", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mmmu_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_mmmu_pro", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mmmu_pro", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro_standard", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mmmu_pro_standard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro_vision", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mmmu_pro_vision", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mtob_full_book", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mtob_full_book", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_mtob_full_book", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mtob_full_book", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mtob_half_book", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_mtob_half_book", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_mtob_half_book", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mtob_half_book", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_needle_in_a_haystack", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_nexus", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_nexus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_nih_multi_needle", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_nih_multi_needle", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_open_rewrite", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_open_rewrite", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_quac", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_quac", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_squad_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_squad_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_textvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tldr9", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_tldr9", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tydiqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_tydiqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": "occurrence_meta_occurrence_meta_llama_4_tydiqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_tydiqa", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ_benchmark_vqav2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "model_id": "model_meta_model_meta_llama_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_aider_polyglot", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_aider_polyglot", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_aime_2025", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_aime_2025", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_descriptive", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_charxiv_descriptive", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_charxiv_descriptive", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_charxiv_descriptive", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_charxiv_reasoning", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_collie", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_collie", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_collie", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_collie", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_complexfuncbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_complexfuncbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_complexfuncbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_complexfuncbench", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_gpqa_diamond", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_graphwalks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_graphwalks", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_ifeval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_ifeval", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_ifeval", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_massive_multitask_language_understanding", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmlu", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_mathvista", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mathvista", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mathvista", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmmlu", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmmu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmmu", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_if", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_multi_if", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_multi_if", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_multi_if", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_multichallenge", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_multichallenge", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_mrcr", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_openai_mrcr", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_openai_mrcr", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_bench_verified", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_lancer", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_swe_lancer", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_lancer", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_lancer", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_lancer_ic_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_lancer_ic_diamond", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau_bench_airline", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_tau_bench_airline", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_tau_bench_airline", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_tau_bench_airline", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau_bench_retail", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_tau_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_tau_bench_retail", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_tau_bench_retail", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_video_mme", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250414_introducing_gpt_4_1_in_the_api_benchmark_video_mme", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "occurrence_id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_video_mme", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_video_mme", "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250416_introducing_openai_o3_and_o4_mini_benchmark_aime_2024", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_aime_2024", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_aime_2024", "release_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250416_introducing_openai_o3_and_o4_mini_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_aime_2025", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_aime_2025", "release_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250416_introducing_openai_o3_and_o4_mini_benchmark_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_charxiv_reasoning", "release_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250416_introducing_openai_o3_and_o4_mini_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_codeforces", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_codeforces", "release_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250416_introducing_openai_o3_and_o4_mini_benchmark_mathvista", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_mathvista", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_mathvista", "release_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250416_introducing_openai_o3_and_o4_mini_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_mmmu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_mmmu", "release_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250416_introducing_openai_o3_and_o4_mini_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_swe_bench_verified", "release_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau_bench_retail", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250416_introducing_openai_o3_and_o4_mini_benchmark_tau_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "occurrence_id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_tau_bench_retail", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_tau_bench_retail", "release_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250417_introducing_gemini_2_5_flash_lmarena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250417_introducing_gemini_2_5_flash_lmarena", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250417_introducing_gemini_2_5_flash_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_activitynet_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_activitynet_qa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_activitynet_qa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_ai2d", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_ai2d", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_ai2d", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_blink", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_blink", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_blink", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cg_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_cg_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_cg_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_cg_bench", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charades_sta", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_charades_sta", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_charades_sta", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_charades_sta", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_chartqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_chartqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_chartqa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cv_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_cv_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_cv_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_cv_bench", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_docvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_docvqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_docvqa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_dream_1k", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_dream_1k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_dream_1k", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_dream_1k", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_egoschema", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_egoschema", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_egoschema", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_eventhallusion", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_eventhallusion", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_event_hallusion", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_event_hallusion", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_flickr30k_captions", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_flickr30k_captions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_flickr30k_captions", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_flickr30k_captions", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infographicsvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_infographicsvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_infographicsvqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_infographicsvqa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_longvideobench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_longvideobench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_longvideobench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_longvideobench", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_lvbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_lvbench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_lvbench", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_mlvu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_mlvu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_mlvu", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_mmmu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_mmmu", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_mmmu", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_motionbench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_motionbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_motionbench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_motionbench", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ms_coco_captions", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_ms_coco_captions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_coco_captions", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_coco_captions", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_mvbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_mvbench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_mvbench", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_next_qa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_next_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_next_qa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_next_qa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nocaps", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_nocaps", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_nocaps", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_nocaps", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocrbench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_ocrbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_ocrbench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_ocrbench", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ok_vqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_ok_vqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_ok_vqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_ok_vqa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_perception_test", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_perception_test", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_perception_test", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_perception_test", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_plm_videobench_fgqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_plm_videobench_fgqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_plm_videobench_fgqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_plm_videobench_fgqa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_plm_videobench_rcap", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_plm_videobench_rcap", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_plm_videobench_rcap", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_plm_videobench_rcap", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_plm_videobench_rdcap", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_plm_videobench_rdcap", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_plm_videobench_rdcap", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_plm_videobench_rdcap", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_plm_videobench_rtloc", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_plm_videobench_rtloc", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_plm_videobench_rtloc", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_plm_videobench_rtloc", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_plm_videobench_sgqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_plm_videobench_sgqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_plm_videobench_sgqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_plm_videobench_sgqa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_pope", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_pope", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_pope", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_pope", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_realworldqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_realworldqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_realworldqa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refcoco", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_refcoco", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_refcoco", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_refcoco", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refcoco_plus", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_refcoco_plus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_refcoco_plus", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_refcoco_plus", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refcocog", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_refcocog", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_refcocog", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_refcocog", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_seed_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_seed_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_seed_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_seed_bench", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_star_video_qa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_star_video_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_star_video_qa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_star_video_qa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tempcompass", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_tempcompass", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_tempcompass", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_tempcompass", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_temporalbench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_temporalbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_temporalbench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_temporalbench", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_textvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_textvqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_textvqa", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tomato", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_tomato", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_tomato", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_tomato", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_video_mme", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_video_mme", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_video_mme", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_video_mme", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videohallucer", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_videohallucer", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_videohallucer", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_videohallucer", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vizwiz", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_vizwiz", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_vizwiz", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_vizwiz", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_vqav2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_vqav2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_vqav2", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vsr", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_derived_20250417_meta_perception_language_model_benchmark_vsr", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "model_id": "model_meta_model_meta_plm", "occurrence_id": "occurrence_meta_occurrence_meta_plm_vsr", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_vsr", "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aider", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aider", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aime_2024", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aime_2024", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aime_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aime_2025", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_arena_hard", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_arena_hard", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_bbh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_bbh", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_berkeley_function_calling_leaderboard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_bfcl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_bfcl", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_codeforces", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_codeforces", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_codeforces", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_crux_o", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_crux_o", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_crux_o", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_crux_o", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_evalplus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_evalplus", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_gpqa", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_gsm8k", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_include", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_include", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_livebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_livebench", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_livecodebench", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_math", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mbpp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mbpp", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mgsm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mgsm", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu_pro", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu_redux", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmmlu", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multiif", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_multiif", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_multiif", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_multiif", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_multipl_e", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_multipl_e", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_supergpqa", "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_arc_c", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_berkeley_function_calling_leaderboard_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_gpqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_ifeval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_massive_multitask_language_understanding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_mgsm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_nexus", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_nexus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_open_rewrite", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_open_rewrite", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tldr9", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon_benchmark_tldr9", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "release_id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_video_mme", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_video_mme", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_webdev_arena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_webdev_arena", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_video_mme", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_video_mme", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_webdev_arena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_webdev_arena", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_internal_swe_tasks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250516_introducing_codex_benchmark_openai_internal_swe_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "model_id": "model_openai_model_openai_20250516_introducing_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20250516_introducing_codex_openai_internal_swe_tasks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250516_introducing_codex_openai_internal_swe_tasks", "release_openai_20250516_introducing_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-16", "release_id": "release_openai_20250516_introducing_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250516_introducing_codex_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "model_id": "model_openai_model_openai_20250516_introducing_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20250516_introducing_codex_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250516_introducing_codex_swe_bench_verified", "release_openai_20250516_introducing_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-16", "release_id": "release_openai_20250516_introducing_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_agieval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_arc_c", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_big_bench_extra_hard", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_big_bench_extra_hard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_bird_sql", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_coco_caption", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_coco_caption", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_countbench_vqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_countbench_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_eclektic", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_eclektic", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_facts_grounding", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_facts_grounding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_flores", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_flores", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_global_mmlu_lite", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_global_mmlu_lite", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_mbpp", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural_questions", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_natural_questions", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ok_vqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_ok_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_perception_test_mcvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_perception_test_mcvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_simpleqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_socialiqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_socialiqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_spatialsense_vqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_spatialsense_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tallyqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_tallyqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_winogrande", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_wmt24", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_wmt24_plus_plus", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_wmt24_plus_plus", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_xquad", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_benchmark_xquad", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_livecodebench", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_livecodebench", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_lmarena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_lmarena", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_mmmu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_mmmu", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_usamo_2025", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_usamo_2025", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_webdev_arena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_webdev_arena", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250522_introducing_claude_4_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_aime", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_aime", "release_anthropic_20250522_introducing_claude_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250522_introducing_claude_4_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_bench", "release_anthropic_20250522_introducing_claude_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250522_introducing_claude_4_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_gpqa_diamond", "release_anthropic_20250522_introducing_claude_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250522_introducing_claude_4_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_mmmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_mmmlu", "release_anthropic_20250522_introducing_claude_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250522_introducing_claude_4_benchmark_mmmu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_mmmu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_mmmu", "release_anthropic_20250522_introducing_claude_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250522_introducing_claude_4_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_swe_bench_verified", "release_anthropic_20250522_introducing_claude_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250522_introducing_claude_4_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_terminal_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_terminal_bench", "release_anthropic_20250522_introducing_claude_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_aider_polyglot", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_aider_polyglot", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_aime", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_aime_2024", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_aime_2024", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_aime_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_aime_2025", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_tau_bench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_tau_bench", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bfcl_v3_multiturn", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_bfcl_v3_multiturn", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_bfcl_v3_multiturn", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_bfcl_v3_multiturn", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_simpleqa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_c_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cnmo_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_cnmo_2024", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_cnmo_2024", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_codeforces", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_codeforces", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_frames", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_frames", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_frames", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_gpqa_diamond", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_hle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_humanity_s_last_exam", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_hmmt_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_hmmt_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_hmmt_2025", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_livecodebench", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250528_deepseek_reasoner", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_mmlu_pro", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_mmlu_pro", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_mmlu_redux", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_mmlu_redux", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_simpleqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_simpleqa", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250528_deepseek_reasoner_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "model_id": "model_deepseek_model_deepseek_r1_0528", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_r1_0528_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_swe_bench_verified", "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "release_id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_agieval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_arc_c", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_big_bench_extra_hard", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_big_bench_extra_hard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_bird_sql", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_coco_caption", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_coco_caption", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_countbench_vqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_countbench_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_eclektic", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_eclektic", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_facts_grounding", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_facts_grounding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_flores", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_flores", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_global_mmlu_lite", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_global_mmlu_lite", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_lmarena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_lmarena", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_mmlu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_mmlu", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_mbpp", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural_questions", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_natural_questions", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ok_vqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_ok_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_perception_test_mcvqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_perception_test_mcvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_simpleqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_socialiqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_socialiqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_spatialsense_vqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_spatialsense_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tallyqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_tallyqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_winogrande", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_wmt24", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_xquad", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_benchmark_xquad", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_medqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai_benchmark_medqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_medgemma_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_medgemma_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai_medqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai_medqa", "release_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "release_id": "release_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "model_id": "model_google_deepmind_google_deepmind_t5gemma_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_drop", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_drop", "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "release_id": "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "model_id": "model_google_deepmind_google_deepmind_t5gemma_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_gsm8k", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_gsm8k", "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "release_id": "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "model_id": "model_google_deepmind_google_deepmind_t5gemma_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_mmlu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_mmlu", "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "release_id": "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_superglue", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_benchmark_superglue", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "model_id": "model_google_deepmind_google_deepmind_t5gemma_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_superglue", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_superglue", "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "release_id": "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2025", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "model_id": "model_qwen_model_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder_swe_bench_verified", "release_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250801_try_deep_think_in_the_gemini_app_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_aime_2025", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_aime_2025", "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "release_id": "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250801_try_deep_think_in_the_gemini_app_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_humanitys_last_exam", "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "release_id": "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_imo_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250801_try_deep_think_in_the_gemini_app_benchmark_imo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_imo_2025", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_imo_2025", "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "release_id": "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250801_try_deep_think_in_the_gemini_app_benchmark_livecodebench_v6", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_livecodebench_v6", "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "release_id": "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_aime", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_aime", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_bias_benchmark_for_question_answering", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_bias_benchmark_for_question_answering", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_bench", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cloningscenarios", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_cloningscenarios", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_cloningscenarios", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_creative_biology", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_creative_biology", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_creative_biology", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_figqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_figqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_figqa", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_gpqa_diamond", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_internal_ai_research_evaluation_suite_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_internal_ai_research_evaluation_suite_1", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_kernel_optimization", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_kernel_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_kernel_optimization", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_kernel_optimization", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_lab_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_lab_bench", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_lab_bench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_protocolqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_protocolqa", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_lab_bench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_seqqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_seqqa", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_llm_training_optimization", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_llm_training_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_llm_training_optimization", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_llm_training_optimization", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_long_form_virology_task_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_long_form_virology_task_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_long_form_virology_task_1", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_long_form_virology_task_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_long_form_virology_task_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_long_form_virology_task_2", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_mmmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_mmmlu", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_mmmu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_mmmu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_mmmu", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_novel_compiler", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_novel_compiler", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_novel_compiler", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quadruped_locomotion", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_quadruped_locomotion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_quadruped_locomotion", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_quadruped_locomotion", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_short_horizon_computational_biology_tasks", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_short_horizon_computational_biology_tasks", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_swe_bench_verified", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_swe_bench_verified_hard_subset", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_swe_bench_verified_hard_subset", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_synthesis_screening_evasion", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_synthesis_screening_evasion", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_synthesis_screening_evasion", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_terminal_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_terminal_bench", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_text_based_reinforcement_learning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_text_based_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_text_based_reinforcement_learning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_text_based_reinforcement_learning", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_time_series_forecasting", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250805_claude_opus_4_1_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_time_series_forecasting", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_time_series_forecasting", "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250805_introducing_gpt_oss_benchmark_aime_2024", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "occurrence_id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_aime_2024", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_aime_2024", "release_openai_20250805_introducing_gpt_oss" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250805_introducing_gpt_oss_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "occurrence_id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_aime_2025", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_aime_2025", "release_openai_20250805_introducing_gpt_oss" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250805_introducing_gpt_oss_benchmark_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "occurrence_id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_tau_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_tau_bench", "release_openai_20250805_introducing_gpt_oss" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250805_introducing_gpt_oss_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "occurrence_id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_codeforces", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_codeforces", "release_openai_20250805_introducing_gpt_oss" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250805_introducing_gpt_oss_benchmark_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "occurrence_id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_healthbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_healthbench", "release_openai_20250805_introducing_gpt_oss" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250805_introducing_gpt_oss_benchmark_hle", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "occurrence_id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_humanitys_last_exam", "release_openai_20250805_introducing_gpt_oss" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250805_introducing_gpt_oss_benchmark_massive_multitask_language_understanding", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "occurrence_id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_mmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_mmlu", "release_openai_20250805_introducing_gpt_oss" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_aider_polyglot", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_aider_polyglot", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_aime_2025", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_aime_2025", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_charxiv_reasoning", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_factscore", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_factscore", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_factscore", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_factscore", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_gpqa_diamond", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_healthbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_healthbench", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_longfact", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_longfact", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_longfact", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_longfact", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_mmmu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_mmmu", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_mmmu_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_mmmu_pro", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_multichallenge", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_multichallenge", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250807_introducing_gpt_5_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "occurrence_id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_swe_bench_verified", "release_openai_20250807_introducing_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_agieval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_arc_c", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbq_ambiguous", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_bbq_ambiguous", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbq_disambiguated", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_bbq_disambiguated", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bold", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_bold", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_commonsenseqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_commonsenseqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_crows_pairs", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_crows_pairs", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_mbpp", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural_questions", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_natural_questions", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openbookqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_openbookqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_realtoxicity", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_realtoxicity", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_socialiqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_socialiqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_toxigen", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_toxigen", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_truthfulqa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_truthfulqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_winobias_type_1_2", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_winobias_type_1_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_winobias_type_2_2", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_winobias_type_2_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogender", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_winogender", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper_benchmark_winogrande", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "release_id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_edit", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_aider_edit", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_aider_polyglot", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_aider_polyglot", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_aime", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_aime_2024", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_aime_2024", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_aime_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_aime_2025", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_arc_c", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_arc_easy", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_browsecomp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_browsecomp", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp_zh", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_browsecomp_zh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_browsecomp_zh", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_browsecomp_zh", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c3", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_c3", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_simpleqa", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_c_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ccpm", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_ccpm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmrc", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_cmrc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cnmo_2024", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_codeforces", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_codeforces", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_cruxeval_i", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_cruxeval_i", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cruxeval_o", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_cruxeval_o", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frames", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_gpqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_gpqa_diamond", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_hle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_humanity_s_last_exam", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_hmmt_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_hmmt_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_hmmt_2025", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval_multilingual", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_humaneval_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_livecodebench", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench_base", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_livecodebench_base", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_mbpp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_mmlu_pro", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_mmlu_pro", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_mmlu_redux", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_mmlu_redux", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_mmmlu_non_english", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_mmmlu_non_english", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_naturalquestions", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_naturalquestions", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_needle_in_a_haystack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_pile_test", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_pile_test", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_piqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_race_high", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_race_high", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_race_middle", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_race_middle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_simpleqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_simpleqa", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_swe_bench_multilingual", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_swe_bench_verified", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_terminal_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminal_bench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminal_bench", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250821_deepseek_v3_1_benchmark_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250821_deepseek_v3_1", "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_aime_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_aime_2025", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_arena_hard_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_arena_hard_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_arena_hard_v2", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_bbh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_bbh", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_berkeley_function_calling_leaderboard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_codeforces", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_crux_o", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_crux_o", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_crux_o", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_crux_o", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_evalplus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_evalplus", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_gpqa", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_gsm8k", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_include", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_include", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_livebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_livebench", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_livecodebench_v6", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_math", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mgsm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mgsm", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu_pro", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu_redux", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmmlu", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_multiif", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_multiif", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ruler", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_ruler", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_ruler", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_ruler", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_supergpqa", "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_benchmark_arc_c", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_arc_challenge", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_arc_challenge", "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_benchmark_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_arc_easy", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_arc_easy", "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_benchmark_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_boolq", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_boolq", "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_hellaswag", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_hellaswag", "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_benchmark_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_piqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_piqa", "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_socialiqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_benchmark_socialiqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_social_iqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_social_iqa", "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_benchmark_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_triviaqa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_triviaqa", "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_code_refactoring_tasks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250915_introducing_upgrades_to_codex_benchmark_openai_code_refactoring_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20250915_introducing_upgrades_to_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_code_refactoring_tasks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_code_refactoring_tasks", "release_openai_20250915_introducing_upgrades_to_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "release_id": "release_openai_20250915_introducing_upgrades_to_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_code_review_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250915_introducing_upgrades_to_codex_benchmark_openai_code_review_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20250915_introducing_upgrades_to_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_code_review_evaluation", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_code_review_evaluation", "release_openai_20250915_introducing_upgrades_to_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "release_id": "release_openai_20250915_introducing_upgrades_to_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_frontend_preference_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250915_introducing_upgrades_to_codex_benchmark_openai_frontend_preference_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20250915_introducing_upgrades_to_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_frontend_preference_evaluation", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_frontend_preference_evaluation", "release_openai_20250915_introducing_upgrades_to_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "release_id": "release_openai_20250915_introducing_upgrades_to_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20250915_introducing_upgrades_to_codex_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20250915_introducing_upgrades_to_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_swe_bench_verified", "release_openai_20250915_introducing_upgrades_to_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "release_id": "release_openai_20250915_introducing_upgrades_to_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_aime_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_aime_2025", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_fleurs", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_fleurs", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_fleurs", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_fleurs", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gtzan", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_gtzan", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_gtzan", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_gtzan", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hallusionbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_hallusionbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_hallusionbench", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_librispeech", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_librispeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_librispeech", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_librispeech", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mathvision", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_minimax_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_minimax_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_minimax_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_minimax_multilingual", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mir_1k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_mir_1k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mir_1k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mir_1k", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mlvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mlvu", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmau", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_mmau", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mmau", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mmau", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mmmu_pro", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_rul_muchomusic", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_rul_muchomusic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_rul_muchomusic", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_rul_muchomusic", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_seed_tts_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_seed_tts_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_seed_tts_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_seed_tts_eval", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_voicebench_commoneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_voicebench_commoneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_voicebench_commoneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_voicebench_commoneval", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wenetspeech", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_wenetspeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_wenetspeech", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_wenetspeech", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_worldsense", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_worldsense", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_worldsense", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_worldsense", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_writingbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_writingbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_writingbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_writingbench", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_zebralogic", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_benchmark_zebralogic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_zebralogic", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_zebralogic", "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_edit", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_aider_edit", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_aider_polyglot", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_aider_polyglot", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_aime", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_arc_c", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_arc_easy", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_browsecomp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_browsecomp", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_browsecomp_zh", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_browsecomp_zh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_browsecomp_zh", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_browsecomp_zh", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_c3", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_c3", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_simpleqa", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_c_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ccpm", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_ccpm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmrc", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_cmrc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cnmo_2024", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_codeforces", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_codeforces", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cruxeval_i", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_cruxeval_i", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cruxeval_o", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_cruxeval_o", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frames", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_gpqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_gpqa_diamond", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_hle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_humanity_s_last_exam", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hmmt_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_hmmt_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval_multilingual", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_humaneval_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_livecodebench", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench_base", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_livecodebench_base", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_mbpp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_mmlu_pro", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_mmlu_pro", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu_non_english", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_mmmlu_non_english", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_naturalquestions", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_naturalquestions", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_needle_in_a_haystack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_pile_test", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_pile_test", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_piqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_race_high", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_race_high", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_race_middle", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_race_middle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_simpleqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_simpleqa", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_swe_bench_multilingual", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_swe_bench_verified", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_terminal_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_terminal_bench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_terminal_bench", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250922_deepseek_v3_1_terminus_benchmark_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250922_deepseek_v3_1_terminus", "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d_test", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ai2d_test", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ai2d_test", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_androidworld", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_androidworld", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arena_hard_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_arena_hard_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_arena_hard_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_arena_hard_v2", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arkitscenes", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_arkitscenes", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_arkitscenes", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_arkitscenes", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_berkeley_function_calling_leaderboard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_bfcl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_bfcl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_blink", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_blink", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_blink", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_cc_ocr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_cc_ocr", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charadessta", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_charadessta", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_charadessta", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_charadessta", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartmimic", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_chartmimic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_chartmimic", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_chartmimic", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_charxiv_rq", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_charxiv_rq", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_countbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_countbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_countbench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_creative_writing_v3", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_creative_writing_v3", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_creative_writing_v3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_creative_writing_v3", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_csimpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_csimpleqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_csimpleqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_csimpleqa", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_design2code", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_design2code", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_design2code", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_design2code", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa_test", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_docvqa_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_docvqa_test", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_docvqa_test", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_embspatialbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_embspatialbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_embspatialbench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_erqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_erqa", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_fleurs", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_fleurs", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gtzan", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_gtzan", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hallusionbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hallusionbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hallusionbench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hle", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hle", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_hmmt_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hmmt_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hmmt_2025", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hypersim", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_hypersim", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hypersim", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hypersim", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ifeval", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_include", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_include", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_infovqa_test", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_infovqa_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_infovqa_test", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_infovqa_test", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_librispeech", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_librispeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livebench_july_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_livebench_july_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livebench_july_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livebench_july_2025", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_v5", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_livecodebench_v5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livecodebench_v5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livecodebench_v5", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livecodebench_v6", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_lvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_lvbench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathverse", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mathverse", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathverse", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathverse", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathvision", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathvista", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathvista", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_minimax_multilingual", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_minimax_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mir_1k", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mir_1k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mlvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mlvu", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mm_mt_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mm_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mm_mt_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mm_mt_bench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmau", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mmau", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mmbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmbench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmbench_en_1_1_dev", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmbench_en_1_1_dev", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlongbench_doc", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mmlongbench_doc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlongbench_doc", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlongbench_doc", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_prox", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlu_prox", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlu_prox", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlu_redux", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmmu", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmmu_pro", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmstar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmstar", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_muirbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_muirbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_muirbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_muirbench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multiif", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_multiif", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_multiif", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_multiif", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_multipl_e", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_multipl_e", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_objectron", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_objectron", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_objectron", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_objectron", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocrbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ocrbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ocrbench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocrbenchv2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_ocrbenchv2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ocrbenchv2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ocrbenchv2", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_odinw13", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_odinw13", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_odinw13", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_osworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_osworld", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_osworld", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_g", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_osworld_g", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_osworld_g", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_osworld_g", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_realworldqa", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refcoco", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_refcoco", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_refcoco", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refspatialbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_refspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_refspatialbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_refspatialbench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_robospatialhome", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_robospatialhome", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_robospatialhome", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_robospatialhome", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_rul_muchomusic", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_rul_muchomusic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_screenspot", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_screenspot", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_screenspot", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_screenspot", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_screenspot_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_screenspot_pro", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_seed_tts_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_seed_tts_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_sifo", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_sifo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sifo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sifo", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_sifo_multiturn", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_sifo_multiturn", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sifo_multiturn", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sifo_multiturn", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_simpleqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_simpleqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_simpleqa", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_simplevqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_simplevqa", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_sunrgbd", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_sunrgbd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sunrgbd", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sunrgbd", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_supergpqa", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_unisvg", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_unisvg", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_unisvg", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_unisvg", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_without_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_videomme_without_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_videomme_without_subtitles", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videommmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_videommmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_videommmu", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_visulogic", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_visulogic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_visulogic", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_visulogic", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_voicebench_commoneval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_voicebench_commoneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vsi_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_vsi_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_vsi_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_vsi_bench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wenetspeech", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_wenetspeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_worldsense", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_worldsense", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_writingbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_writingbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_writingbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_writingbench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_zebralogic", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_zebralogic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_zerobench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_zerobench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_zerobench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_zerobench", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_zerobench_sub", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_benchmark_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_zerobench_sub", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_zerobench_sub", "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_aime_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_aime_2025", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_arena_hard_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_berkeley_function_calling_leaderboard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_codeforces", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_crux_o", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_crux_o", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_gpqa", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_hmmt_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_hmmt_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_hmmt_2025", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livebench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_livecodebench_v6", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multiif", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_multiif", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ruler", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_ruler", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_supergpqa", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_swe_bench_verified", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau2_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_tau2_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_tau2_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_tau2_bench", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_text_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_benchmark_text_arena", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "occurrence_id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_text_arena", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_text_arena", "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agent_red_teaming", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_agent_red_teaming", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_agent_red_teaming", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_agent_red_teaming", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agentic_misalignment_suite", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_agentic_misalignment_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_agentic_misalignment_suite", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_agentic_misalignment_suite", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_ai2d", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_aime", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_aime", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_bbh", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_bias_benchmark_for_question_answering", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_bias_benchmark_for_question_answering", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_blackmail_suite", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_blackmail_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_blackmail_suite", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_blackmail_suite", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_chartqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cloningscenarios", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cloningscenarios", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cloningscenarios", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_computer_use_prompt_injection_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_computer_use_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_computer_use_prompt_injection_evaluation", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_computer_use_prompt_injection_evaluation", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_creative_biology", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_creative_biology", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_creative_biology", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cybench", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cybergym", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cybergym", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_dna_synthesis_screening_evasion", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_dna_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_dna_synthesis_screening_evasion", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_dna_synthesis_screening_evasion", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_docvqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_drop", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_false_premise_questions", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_false_premise_questions", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_false_premise_questions", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_false_premise_questions", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_figqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_figqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_figqa", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_financial_crimes", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_financial_crimes", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_financial_crimes", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_financial_crimes", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_gsm8k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_humaneval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_impossible_tasks", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_impossible_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_impossible_tasks", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_impossible_tasks", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_incalmo_cyber_ranges", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_incalmo_cyber_ranges", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_incalmo_cyber_ranges", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_incalmo_cyber_ranges", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_internal_ai_research_evaluation_suite_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_internal_ai_research_evaluation_suite_1", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_internal_ai_research_evaluation_suite_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_internal_ai_research_evaluation_suite_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_internal_ai_research_evaluation_suite_2", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_kernels_task", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_kernels_task", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_kernels_task", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_kernels_task", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_lab_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_lab_bench", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_lab_bench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_protocolqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_protocolqa", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_lab_bench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_seqqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_seqqa", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_llm_training", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_llm_training", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_llm_training", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_llm_training", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_long_form_virology_task_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_long_form_virology_task_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_long_form_virology_task_1", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_long_form_virology_task_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_long_form_virology_task_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_long_form_virology_task_2", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mathvista", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_prompt_injection_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mcp_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mcp_prompt_injection_evaluation", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mcp_prompt_injection_evaluation", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mgsm", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mmmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mmmlu", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_needle_in_a_haystack", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_novel_compiler", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_novel_compiler", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_novel_compiler", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_opinion_elicitation", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_opinion_elicitation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_opinion_elicitation", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_opinion_elicitation", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_osworld", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_osworld", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_pretraining_documents", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_pretraining_documents", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_pretraining_documents", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_pretraining_documents", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quadruped_reinforcement_learning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_quadruped_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_quadruped_reinforcement_learning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_quadruped_reinforcement_learning", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_random_chat_questions", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_random_chat_questions", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_random_chat_questions", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_random_chat_questions", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_100q_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_record_100q_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_x_100q_hard", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_x_100q_hard", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_2_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_record_2_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_x_2_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_x_2_bench", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_research_sabotage", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_research_sabotage", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_research_sabotage", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_research_sabotage", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_research_sandbagging", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_research_sandbagging", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_research_sandbagging", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_research_sandbagging", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_reward_hack_prone_coding_tasks_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_reward_hack_prone_coding_tasks_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_reward_hack_prone_coding_tasks_v2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_reward_hack_prone_coding_tasks_v2", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_shade_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_shade_arena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_shade_arena", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_short_horizon_computational_biology_tasks", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_short_horizon_computational_biology_tasks", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_simpleqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_simpleqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_simpleqa", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_subversion_strategy", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_subversion_strategy", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_subversion_strategy", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_subversion_strategy", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_swe_bench_verified", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_swe_bench_verified_hard_subset", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_swe_bench_verified_hard_subset", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_sycophancy_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_sycophancy_eval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_sycophancy_eval", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_sycophancy_eval", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_synthetic_honeypots", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_synthetic_honeypots", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_synthetic_honeypots", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_synthetic_honeypots", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_terminal_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_terminal_bench", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_text_based_reinforcement_learning_task", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_text_based_reinforcement_learning_task", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_text_based_reinforcement_learning_task", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_text_based_reinforcement_learning_task", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_time_series_forecasting", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_time_series_forecasting", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_time_series_forecasting", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tool_use_prompt_injection_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_tool_use_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_tool_use_prompt_injection_evaluation", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_tool_use_prompt_injection_evaluation", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vals_ai_finance_agent", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_vals_ai_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_vals_ai_finance_agent", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_vals_ai_finance_agent", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vmqa_vct", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_vmqa_vct", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_vmqa_vct", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_vmqa_vct", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wildchat", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_wildchat", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_xstest", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_xstest", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_edit", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_aider_edit", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_aider_polyglot", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_aider_polyglot", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_aime", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_aime_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_aime_2025", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_arc_c", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_arc_easy", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_browsecomp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_browsecomp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_browsecomp_zh", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_browsecomp_zh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_browsecomp_zh", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_browsecomp_zh", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_c3", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_c3", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_simpleqa", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_c_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ccpm", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_ccpm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmrc", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_cmrc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cnmo_2024", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_codeforces", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_codeforces", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cruxeval_i", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_cruxeval_i", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cruxeval_o", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_cruxeval_o", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frames", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_gpqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_gpqa_diamond", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_hle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_humanity_s_last_exam", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hmmt_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_hmmt_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_hmmt_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_hmmt_2025", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval_multilingual", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_humaneval_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_livecodebench", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench_base", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_livecodebench_base", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_mbpp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_mmlu_pro", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_mmlu_pro", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu_non_english", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_mmmlu_non_english", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_naturalquestions", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_naturalquestions", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_needle_in_a_haystack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_pile_test", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_pile_test", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_piqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_race_high", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_race_high", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_race_middle", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_race_middle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_simpleqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_simpleqa", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_swe_bench_multilingual", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_swe_bench_verified", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_terminal_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_terminal_bench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_terminal_bench", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20250929_deepseek_v3_2_exp_benchmark_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20250929_deepseek_v3_2_exp", "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_benchmark_androidworld", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_computer_use", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_androidworld", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_androidworld", "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "release_id": "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_online_mind2web", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_benchmark_online_mind2web", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_computer_use", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_online_mind2web", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_online_mind2web", "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "release_id": "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_webvoyager", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_benchmark_webvoyager", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_computer_use", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_webvoyager", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_webvoyager", "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "release_id": "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agent_red_teaming", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_agent_red_teaming", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_agent_red_teaming", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_agent_red_teaming", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agentic_misalignment_suite", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_agentic_misalignment_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_agentic_misalignment_suite", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_agentic_misalignment_suite", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_aime", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_aime", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_bias_benchmark_for_question_answering", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_bias_benchmark_for_question_answering", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cloningscenarios", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_cloningscenarios", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_cloningscenarios", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_cybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_cybench", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_figqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_figqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_figqa", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_impossible_tasks", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_impossible_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_impossible_tasks", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_impossible_tasks", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_internal_prompt_injection_evaluations", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_internal_prompt_injection_evaluations", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_internal_prompt_injection_evaluations", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_internal_prompt_injection_evaluations", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_lab_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_lab_bench", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_lab_bench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_protocolqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_protocolqa", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_lab_bench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_seqqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_seqqa", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_long_form_virology_task_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_long_form_virology_task_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_long_form_virology_task_1", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_long_form_virology_task_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_long_form_virology_task_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_long_form_virology_task_2", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_mmmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_mmmlu", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_osworld", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_osworld", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_2_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_record_2_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_x_2_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_x_2_bench", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_reward_hack_prone_coding_tasks_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_reward_hack_prone_coding_tasks_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_reward_hack_prone_coding_tasks_v2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_reward_hack_prone_coding_tasks_v2", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_shade_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_shade_arena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_shade_arena", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_swe_bench_verified", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_swe_bench_verified_hard_subset", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_swe_bench_verified_hard_subset", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_synthesis_screening_evasion", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_synthesis_screening_evasion", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_synthesis_screening_evasion", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_terminal_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_terminal_bench", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vmqa_vct", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_vmqa_vct", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_vmqa_vct", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_vmqa_vct", "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": "occurrence_openai_occurrence_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_aime_2025", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_aime_2025", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": "occurrence_openai_occurrence_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_codeforces", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_codeforces", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_factscore", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_factscore", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_longfact", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_longfact", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_arc_agi_2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_arc_agi_2", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_gpqa_diamond", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_humanitys_last_exam", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_lmarena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_lmarena", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_matharena_apex", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_matharena_apex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_matharena_apex", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_matharena_apex", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_mmmu_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_mmmu_pro", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_simpleqa_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_simpleqa_verified", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_simpleqa_verified", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_swe_bench_verified", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_terminal_bench_2_0", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_terminal_bench_2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_terminal_bench_2", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2025", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_vending_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_vending_bench_2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_vending_bench_2", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_video_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_video_mmmu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_video_mmmu", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_webdev_arena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_webdev_arena", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_code_refactoring_tasks", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251119_building_more_with_gpt_5_1_codex_max_benchmark_openai_code_refactoring_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251119_building_more_with_gpt_5_1_codex_max", "release_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "release_id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_code_review_evaluation", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251119_building_more_with_gpt_5_1_codex_max_benchmark_openai_code_review_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251119_building_more_with_gpt_5_1_codex_max", "release_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "release_id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_frontend_preference_evaluation", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251119_building_more_with_gpt_5_1_codex_max_benchmark_openai_frontend_preference_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251119_building_more_with_gpt_5_1_codex_max", "release_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "release_id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251119_building_more_with_gpt_5_1_codex_max_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "occurrence_id": "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_swe_bench_verified", "release_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "release_id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_swe", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251119_building_more_with_gpt_5_1_codex_max_benchmark_swe_lancer_ic_swe", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "occurrence_id": "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_swe_lancer_ic_swe", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_swe_lancer_ic_swe", "release_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "release_id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251119_building_more_with_gpt_5_1_codex_max_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "occurrence_id": "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_terminal_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_terminal_bench", "release_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "release_id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aider_polyglot", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aider_polyglot", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aider_polyglot", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aime_2025", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aime_2025", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aime_2025", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_arc_agi_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_arc_agi_1", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_arc_agi_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_arc_agi_2", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp_plus", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_browsecomp_plus", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_browsecomp_plus", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_browsecomp_plus", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cloningscenarios", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_creative_biology", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cybench", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cybergym", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cybergym", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_figqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_finance_agent", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_finance_agent", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_gpqa_diamond", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_hle", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_humanity_s_last_exam", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_kernel_optimization", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_kernel_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_figqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_lab_bench_figqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_lab_bench_figqa", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_lab_bench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_lab_bench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_llm_training_optimization", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_llm_training_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_1", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mcp_atlas", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mcp_atlas", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mmmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mmmlu", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mmmu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mmmu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mmmu", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_novel_compiler", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_osworld", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_osworld", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quadruped_locomotion", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_quadruped_locomotion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_airline_corrected", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_record_2_bench_airline_corrected", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_x_2_bench_airline_corrected", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_x_2_bench_airline_corrected", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_2_bench_airline_original", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_record_2_bench_airline_original", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_x_2_bench_airline_original", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_x_2_bench_airline_original", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_spreadsheetbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_spreadsheetbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_spreadsheetbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_spreadsheetbench", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_multilingual", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_pro", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_verified", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_synthesis_screening_evasion", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_terminal_bench_2_0", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_text_based_reinforcement_learning", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_text_based_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_time_series_forecasting", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20251124_introducing_claude_opus_4_5", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_vending_bench_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_vending_bench_2", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_webarena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20251124_introducing_claude_opus_4_5_benchmark_webarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_webarena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_webarena", "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_edit", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_aider_edit", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_aime", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2024", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_aime_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_aime_2025", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_alpacaeval_2_0", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_arc_c", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_arc_easy", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_browsecomp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_browsecomp", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_browsecomp_zh", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_browsecomp_zh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_browsecomp_zh", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_browsecomp_zh", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_c3", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_c3", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_simpleqa", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_c_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ccpm", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_ccpm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmrc", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_cmrc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cnmo_2024", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_codeforces", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_codeforces", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cruxeval_i", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_cruxeval_i", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cruxeval_o", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_cruxeval_o", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frames", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 4, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_gpqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_gpqa_diamond", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_hle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_humanity_s_last_exam", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hmmt_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_hmmt_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_feb_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_hmmt_feb_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_hmmt_feb_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_hmmt_feb_2025", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_nov_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_hmmt_nov_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_hmmt_nov_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_hmmt_nov_2025", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval_multilingual", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_humaneval_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_imoanswerbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_imoanswerbench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_imoanswerbench", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_livecodebench", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench_base", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_livecodebench_base", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math_500", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_mbpp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_mark", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_mcp_mark", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_mcp_mark", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_mcp_mark", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_universe", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_mcp_universe", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_mcp_universe", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_mcp_universe", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_mmlu_pro", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_mmlu_pro", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 3, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu_non_english", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_mmmlu_non_english", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_naturalquestions", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_naturalquestions", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_needle_in_a_haystack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_pile_test", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_pile_test", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_piqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_race_high", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_race_high", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_race_middle", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_race_middle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_swe_bench_multilingual", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_swe_bench_verified", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_tau2_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_tau2_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_tau2_bench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_tau2_bench", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_terminal_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_terminal_bench_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_terminal_bench_2_0", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tool_decathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_tool_decathlon", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_tool_decathlon", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_tool_decathlon", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 5, "id": "status_release_deepseek_20251201_deepseek_v3_2_benchmark_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "model_id": "model_deepseek_model_deepseek_v3_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20251201_deepseek_v3_2", "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_aime_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_aime_2025", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmo_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_cmo_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_cmo_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_cmo_2025", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_codeforces", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_codeforces", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_gpqa_diamond", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_hle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_humanity_s_last_exam", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_feb_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_hmmt_feb_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_hmmt_feb_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_hmmt_feb_2025", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_nov_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_hmmt_nov_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_hmmt_nov_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_hmmt_nov_2025", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_icpc_wf_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_icpc_wf_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_icpc_wf_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_icpc_wf_2025", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_imo_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_imo_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_imo_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_imo_2025", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_imoanswerbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_imoanswerbench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_imoanswerbench", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ioi_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_ioi_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_ioi_2025", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_ioi_2025", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20251201_deepseek_v3_2_speciale_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_livecodebench", "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d_test", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_aime_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_aime_2025", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_hard_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_arena_hard_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arkitscenes", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_arkitscenes", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_berkeley_function_calling_leaderboard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_blink", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_charadessta", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_charadessta", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartmimic", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_chartmimic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_countbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_creative_writing_v3", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_creative_writing_v3", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_csimpleqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_csimpleqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_design2code", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_design2code", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa_test", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_docvqa_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_embspatialbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_fleurs", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_fleurs", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_fleurs", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_fleurs", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_gtzan", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_gtzan", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_gtzan", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_gtzan", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_hallusionbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_hmmt_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hypersim", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_hypersim", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_infovqa_test", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_infovqa_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_librispeech", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_librispeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_livebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_livebench", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livebench_july_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_livebench_july_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench_v5", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_livecodebench_v5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_livecodebench_v6", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathverse", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mathverse", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mathvision", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_minimax_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_minimax_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_minimax_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_minimax_multilingual", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_mir_1k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mir_1k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mir_1k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mir_1k", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mlvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mlvu", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mm_mt_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mm_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmau", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mmau", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmau", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmau", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_mmbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mmbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlongbench_doc", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mmlongbench_doc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_prox", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmmu", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmmu_pro", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_muirbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_muirbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multiif", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_multiif", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_multipl_e", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_multipl_e", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_objectron", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_objectron", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ocrbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ocrbenchv2", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_ocrbenchv2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_odinw13", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_osworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_g", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_osworld_g", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_refcoco", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_refspatialbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_refspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_robospatialhome", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_robospatialhome", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_rul_muchomusic", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_rul_muchomusic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_rul_muchomusic", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_rul_muchomusic", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_screenspot", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_screenspot", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_seed_tts_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_seed_tts_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_seed_tts_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_seed_tts_eval", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_sifo", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_sifo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_sifo_multiturn", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_sifo_multiturn", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_simpleqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_sunrgbd", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_sunrgbd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_unisvg", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_unisvg", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme_without_subtitles", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_videommmu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_visulogic", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_visulogic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_voicebench_commoneval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_voicebench_commoneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_voicebench_commoneval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_voicebench_commoneval", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_vsi_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_vsi_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_wenetspeech", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_wenetspeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_worldsense", "consecutive_reviewed_omissions": 2, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_worldsense", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_writingbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_writingbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_writingbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_writingbench", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_zebralogic", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_zebralogic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_zebralogic", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_zebralogic", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_zerobench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_zerobench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_zerobench_sub", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_benchmark_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251211_introducing_gpt_5_2", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_aime_2025", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_aime_2025", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_arc_agi_1", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_arc_agi_1", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_arc_agi_1", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_arc_agi_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_arc_agi_2", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_browsecomp", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_browsecomp", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp_long_context", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_browsecomp_long_context", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_browsecomp_long_context", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_browsecomp_long_context", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_charxiv_reasoning", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251211_introducing_gpt_5_2", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_factscore", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_factscore", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251211_introducing_gpt_5_2", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontiermath", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_frontiermath", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_frontiermath", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_gdpval", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_gdpval", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_gpqa_diamond", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_graphwalks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_graphwalks", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251211_introducing_gpt_5_2", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_hle", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_humanitys_last_exam", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_hmmt_february_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_hmmt_february_2025", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_hmmt_february_2025", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_longfact", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_longfact", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251211_introducing_gpt_5_2", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_mcp_atlas", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_scale_mcp_atlas", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_scale_mcp_atlas", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_mmmlu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_mmmlu", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251211_introducing_gpt_5_2", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_mmmu_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_mmmu_pro", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251211_introducing_gpt_5_2", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_chatgpt_factuality", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_openai_chatgpt_factuality", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_chatgpt_factuality", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_chatgpt_factuality", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_investment_banking_spreadsheet_tasks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_openai_investment_banking_spreadsheet_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_investment_banking_spreadsheet_tasks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_investment_banking_spreadsheet_tasks", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_mrcr", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_mrcr", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_mrcr", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_record_2_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_tau2_bench_retail", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_tau2_bench_retail", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_2_bench_telecom", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_record_2_bench_telecom", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_tau2_bench_telecom", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_tau2_bench_telecom", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_screenspot_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_screenspot_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_screenspot_pro", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_bench_pro", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_bench_verified", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_lancer_ic_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_lancer_ic_diamond", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_toolathlon", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_toolathlon", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251211_introducing_gpt_5_2_benchmark_video_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "occurrence_id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_video_mmmu", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_video_mmmu", "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_gpqa_diamond", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_humanitys_last_exam", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_lmarena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_lmarena", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_matharena_apex", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_matharena_apex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_mmmu_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_mmmu_pro", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_simpleqa_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_swe_bench_verified", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_terminal_bench_2_0", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2025", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_vending_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_video_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_code_refactoring_tasks", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20251218_introducing_gpt_5_2_codex_benchmark_openai_code_refactoring_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251218_introducing_gpt_5_2_codex", "release_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_code_review_evaluation", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20251218_introducing_gpt_5_2_codex_benchmark_openai_code_review_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251218_introducing_gpt_5_2_codex", "release_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_frontend_preference_evaluation", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20251218_introducing_gpt_5_2_codex_benchmark_openai_frontend_preference_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251218_introducing_gpt_5_2_codex", "release_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_professional_ctf", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251218_introducing_gpt_5_2_codex_benchmark_professional_ctf", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_professional_ctf", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_professional_ctf", "release_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251218_introducing_gpt_5_2_codex_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_swe_bench_pro", "release_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251218_introducing_gpt_5_2_codex_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251218_introducing_gpt_5_2_codex", "release_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_swe", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20251218_introducing_gpt_5_2_codex_benchmark_swe_lancer_ic_swe", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20251218_introducing_gpt_5_2_codex", "release_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20251218_introducing_gpt_5_2_codex_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_terminal_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_terminal_bench", "release_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_aa_lcr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_aa_lcr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_aa_lcr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_aa_lcr", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arena_hard_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_arena_hard_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_arena_hard_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_arena_hard_v2", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bfcl_v4", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_bfcl_v4", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_bfcl_v4", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_bfcl_v4", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_c_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_c_eval", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deep_planning", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_deep_planning", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_deep_planning", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_gpqa", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hle", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hle", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle_with_tools", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hle_with_tools", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hle_with_tools", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hmmt_february_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hmmt_february_2025", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_november_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hmmt_november_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hmmt_november_2025", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_ifbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_ifbench", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_imoanswerbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_imoanswerbench", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_livecodebench_v6", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_mmlu_pro", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_mmlu_redux", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_multichallenge", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_multichallenge", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_multichallenge", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_swe_bench_verified", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau2_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_tau2_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_tau2_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_tau2_bench", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vita_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260126_qwen3_max_thinking_benchmark_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_vita_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_vita_bench", "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260203_qwen3_coder_next_benchmark_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_aider", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_aider", "release_qwen_20260203_qwen3_coder_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "release_id": "release_qwen_20260203_qwen3_coder_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260203_qwen3_coder_next_benchmark_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_multilingual", "release_qwen_20260203_qwen3_coder_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "release_id": "release_qwen_20260203_qwen3_coder_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260203_qwen3_coder_next_benchmark_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_pro", "release_qwen_20260203_qwen3_coder_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "release_id": "release_qwen_20260203_qwen3_coder_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260203_qwen3_coder_next_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_verified", "release_qwen_20260203_qwen3_coder_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "release_id": "release_qwen_20260203_qwen3_coder_next", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260203_qwen3_coder_next_benchmark_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_terminal_bench_2_0", "release_qwen_20260203_qwen3_coder_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "release_id": "release_qwen_20260203_qwen3_coder_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_aider_polyglot", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_aime_2025", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_aime_2025", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_aime_2025", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_arc_agi_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_arc_agi_1", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_arc_agi_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_arc_agi_2", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_biglaw_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biglaw_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biglaw_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biglaw_bench", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biomysterybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biomysterybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biomysterybench", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biopipelinebench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biopipelinebench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biopipelinebench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biopipelinebench", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_browsecomp", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_browsecomp", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp_plus", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_browsecomp_plus", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_charxiv_reasoning", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cloningscenarios", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_creative_biology", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cybench", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cybergym", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cybergym", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deepsearchqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_deepsearchqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_deepsearchqa", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_figqa", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_finance_agent", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_finance_agent", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_gdpval_aa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_gdpval_aa", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_gpqa_diamond", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_graphwalks", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_graphwalks", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_hle", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_humanity_s_last_exam", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_kernel_optimization", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_kernel_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_figqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_lab_bench_figqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_lab_bench_figqa", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_lab_bench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_lab_bench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_llm_training_optimization", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_llm_training_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_1", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_2", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mcp_atlas", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mcp_atlas", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmlu", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmu_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmu_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmu_pro", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_novel_compiler", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_mrcr_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_openai_mrcr_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_openai_mrcr_v2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_openai_mrcr_v2", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openrca", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_openrca", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_openrca", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_openrca", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_osworld_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_osworld_verified", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quadruped_locomotion", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_quadruped_locomotion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_real_world_finance", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_real_world_finance", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_real_world_finance", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_real_world_finance", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_2_bench_airline_corrected", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_record_2_bench_airline_corrected", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_airline_original", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_record_2_bench_airline_original", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_record_2_bench_retail", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_x_2_bench_retail", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_x_2_bench_retail", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_2_bench_telecom", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_record_2_bench_telecom", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_x_2_bench_telecom", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_x_2_bench_telecom", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_spreadsheetbench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_spreadsheetbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_multilingual", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_verified", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_synthesis_screening_evasion", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_terminal_bench_2_0", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_text_based_reinforcement_learning", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_text_based_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_time_series_forecasting", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260205_introducing_claude_opus_4_6", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_vending_bench_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_vending_bench_2", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_webarena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260205_introducing_claude_opus_4_6_benchmark_webarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_webarena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_webarena", "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cybersecurity_ctf", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_cybersecurity_ctf", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_cybersecurity_ctf", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_cybersecurity_ctf", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_gdpval", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_gdpval", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_code_refactoring_tasks", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_openai_code_refactoring_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260205_introducing_gpt_5_3_codex", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_code_review_evaluation", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_openai_code_review_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260205_introducing_gpt_5_3_codex", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_frontend_preference_evaluation", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_openai_frontend_preference_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260205_introducing_gpt_5_3_codex", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_osworld_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_osworld_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_osworld_verified", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_professional_ctf", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_professional_ctf", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260205_introducing_gpt_5_3_codex", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_swe_bench_pro", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260205_introducing_gpt_5_3_codex", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_swe_lancer_ic_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_swe_lancer_ic_diamond", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_swe", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_swe_lancer_ic_swe", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260205_introducing_gpt_5_3_codex", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260205_introducing_gpt_5_3_codex_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "occurrence_id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_terminal_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_terminal_bench", "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_deep_think_for_science", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-11", "release_id": "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_deep_think_for_science", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-11", "release_id": "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_imo_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w_benchmark_imo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_deep_think_for_science", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-11", "release_id": "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w_benchmark_livecodebench_v6", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_deep_think_for_science", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-11", "release_id": "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_benchmark_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_arc_agi_2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_arc_agi_2", "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmt_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_benchmark_cmt_benchmark", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_cmt_benchmark", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_cmt_benchmark", "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_benchmark_codeforces", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_codeforces", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_codeforces", "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_humanitys_last_exam", "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_imo_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_benchmark_imo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_imo_2025", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_imo_2025", "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_international_chemistry_olympiad_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_benchmark_international_chemistry_olympiad_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_icho_2025", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_icho_2025", "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_international_physics_olympiad_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_benchmark_international_physics_olympiad_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_ipho_2025", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_ipho_2025", "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_benchmark_livecodebench_v6", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_internal_swe_tasks", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260212_introducing_gpt_5_3_codex_spark_benchmark_openai_internal_swe_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "model_id": "model_openai_model_openai_20260212_introducing_gpt_5_3_codex_spark", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260212_introducing_gpt_5_3_codex_spark", "release_openai_20260212_introducing_gpt_5_3_codex_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_openai_20260212_introducing_gpt_5_3_codex_spark", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260212_introducing_gpt_5_3_codex_spark_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "model_id": "model_openai_model_openai_20260212_introducing_gpt_5_3_codex_spark", "occurrence_id": "occurrence_openai_occurrence_openai_20260212_introducing_gpt_5_3_codex_spark_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260212_introducing_gpt_5_3_codex_spark_swe_bench_pro", "release_openai_20260212_introducing_gpt_5_3_codex_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_openai_20260212_introducing_gpt_5_3_codex_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260212_introducing_gpt_5_3_codex_spark_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "model_id": "model_openai_model_openai_20260212_introducing_gpt_5_3_codex_spark", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260212_introducing_gpt_5_3_codex_spark", "release_openai_20260212_introducing_gpt_5_3_codex_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_openai_20260212_introducing_gpt_5_3_codex_spark", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260212_introducing_gpt_5_3_codex_spark_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "model_id": "model_openai_model_openai_20260212_introducing_gpt_5_3_codex_spark", "occurrence_id": "occurrence_openai_occurrence_openai_20260212_introducing_gpt_5_3_codex_spark_terminal_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260212_introducing_gpt_5_3_codex_spark_terminal_bench", "release_openai_20260212_introducing_gpt_5_3_codex_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_openai_20260212_introducing_gpt_5_3_codex_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aa_lcr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_aa_lcr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_aa_lcr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_aa_lcr", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d_test", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ai2d_test", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ai2d_test", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_aime_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_aime_2026", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_aime_2026", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_androidworld", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_androidworld", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_babyvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_babyvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_babyvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_babyvision", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_bbh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_bbh", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bfcl_v4", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_bfcl_v4", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_bfcl_v4", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_bfcl_v4", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_browsecomp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_browsecomp", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_browsecomp", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp_zh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_browsecomp_zh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_browsecomp_zh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_browsecomp_zh", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_c_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_c_eval", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_cc_ocr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_cc_ocr", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_charxiv_rq", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_charxiv_rq", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_countbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_countbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_countbench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_crux_i", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_crux_i", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_crux_i", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_crux_i", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_crux_o", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_crux_o", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_crux_o", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_crux_o", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deep_planning", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_deep_planning", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_deep_planning", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_dynamath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_dynamath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_dynamath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_dynamath", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_embspatialbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_embspatialbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_embspatialbench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_erqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_erqa", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_evalplus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_evalplus", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_global_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_global_piqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_global_piqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_global_piqa", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_gpqa", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_gsm8k", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hallusionbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hallusionbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hallusionbench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_hle_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle_verified", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle_with_tools", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle_with_tools", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle_with_tools", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hmmt_february_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hmmt_february_2025", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_november_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hmmt_november_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hmmt_november_2025", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hypersim", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_hypersim", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hypersim", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hypersim", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ifbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ifbench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ifeval", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_imoanswerbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_imoanswerbench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_include", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_include", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_korbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_korbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_korbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_korbench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lingoqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_lingoqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_lingoqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_lingoqa", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_livecodebench_v6", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_longbench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_longbench_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_longbench_v2", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_lvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_lvbench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_math", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mathvision", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista_mini", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mathvista_mini", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mathvista_mini", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mathvista_mini", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_maxife", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_maxife", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_maxife", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_maxife", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_mark", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mcp_mark", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mcp_mark", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_medxpertqa_mm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_medxpertqa_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_medxpertqa_mm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_medxpertqa_mm", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mlvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mlvu", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmbench_en_1_1_dev", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmbench_en_1_1_dev", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlongbench_doc", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mmlongbench_doc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlongbench_doc", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlongbench_doc", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_pro", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_prox", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_prox", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_prox", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_redux", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmlu", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmu", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmu_pro", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmstar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmstar", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mmvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmvu", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_multichallenge", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_multichallenge", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_multichallenge", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_multipl_e", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_multipl_e", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mvbench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nova", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_nova", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_nova", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_nova", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nuscene", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_nuscene", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_nuscene", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_nuscene", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocrbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ocrbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ocrbench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_odinw13", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_odinw13", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_odinw13", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_omnidocbench_1_5", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_omnidocbench_1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_omnidocbench_1_5", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_osworld_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_osworld_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_osworld_verified", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_pmc_vqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_pmc_vqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_pmc_vqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_pmc_vqa", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_polymath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_polymath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_polymath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_polymath", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_realworldqa", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refcoco", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_refcoco", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_refcoco", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refspatialbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_refspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_refspatialbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_refspatialbench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_screenspot_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_screenspot_pro", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_seal_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_seal_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_seal_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_seal_0", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_seccodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_seccodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_seccodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_seccodebench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_simplevqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_simplevqa", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_slake", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_slake", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_slake", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_slake", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_sunrgbd", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_sunrgbd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_sunrgbd", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_sunrgbd", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_supergpqa", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_agentless", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_swe_agentless", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_agentless", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_agentless", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_bench_multilingual", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_bench_verified", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau2_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_tau2_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_tau2_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_tau2_bench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_terminal_bench_2_0", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tool_decathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_tool_decathlon", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_tool_decathlon", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_tool_decathlon", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_v", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_v", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_v_star", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_v_star", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_with_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videomme_with_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videomme_with_subtitles", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_without_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videomme_without_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videomme_without_subtitles", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videommmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videommmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videommmu", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vita_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_vita_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_vita_bench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_we_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_we_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_we_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_we_math", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_widesearch", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_widesearch", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_widesearch", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_wmt24", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_wmt24_plus_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_wmt24_plus_plus", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_zerobench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_zerobench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_zerobench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_zerobench", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_zerobench_sub", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260216_qwen3_5_benchmark_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_zerobench_sub", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_zerobench_sub", "release_qwen_20260216_qwen3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agent_red_teaming", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_agent_red_teaming", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agentic_misalignment_suite", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_agentic_misalignment_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_ai2d", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_aime_2025", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_aime_2025", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_aime_2025", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_arc_agi_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_arc_agi_1", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_arc_agi_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_arc_agi_2", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_bbh", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_biomysterybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_biomysterybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_biomysterybench", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biopipelinebench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_biopipelinebench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_biopipelinebench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_biopipelinebench", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_blackmail_suite", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_blackmail_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_browsecomp", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_browsecomp", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_chartqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_charxiv_reasoning", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cloningscenarios", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_computer_use_prompt_injection_evaluation", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_computer_use_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_creative_biology", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cybench", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cybergym", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cybergym", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deepsearchqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_deepsearchqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_deepsearchqa", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_dna_synthesis_screening_evasion", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_dna_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_docvqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_drop", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_false_premise_questions", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_false_premise_questions", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_figqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_finance_agent", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_finance_agent", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_financial_crimes", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_financial_crimes", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_financial_services_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_financial_services_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_financial_services_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_financial_services_benchmark", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gdpval_aa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gdpval_aa", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_global_mmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_global_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_global_mmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_global_mmlu", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gpqa_diamond", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_graphwalks", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_graphwalks", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gsm8k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_hle", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_humanity_s_last_exam", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_humaneval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_impossible_tasks", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_impossible_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_incalmo_cyber_ranges", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_incalmo_cyber_ranges", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_internal_ai_research_evaluation_suite_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_kernels_task", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_kernels_task", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_figqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_lab_bench_figqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_lab_bench_figqa", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_lab_bench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_lab_bench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_llm_training", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_llm_training", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_1", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mathvista", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mcp_atlas", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mcp_atlas", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_prompt_injection_evaluation", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mcp_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_medcalc_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_medcalc_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_medcalc_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_medcalc_bench_verified", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mgsm", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_milu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_milu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_milu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_milu", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mmmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mmmlu", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mmmu_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mmmu_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mmmu_pro", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_needle_in_a_haystack", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_novel_compiler", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_officeqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_officeqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_officeqa", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_mrcr_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_openai_mrcr_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_openai_mrcr_v2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_openai_mrcr_v2", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openrca", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_openrca", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_openrca", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_openrca", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_opinion_elicitation", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_opinion_elicitation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_osworld_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_osworld_verified", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_pretraining_documents", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_pretraining_documents", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_quadruped_reinforcement_learning", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_quadruped_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_random_chat_questions", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_random_chat_questions", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_real_world_finance", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_real_world_finance", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_real_world_finance", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_real_world_finance", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_100q_hard", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_record_100q_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_record_2_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_record_2_bench_retail", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_x_2_bench_retail", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_x_2_bench_retail", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_research_sabotage", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_research_sabotage", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_research_sandbagging", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_research_sandbagging", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_reward_hack_prone_coding_tasks_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_reward_hack_prone_coding_tasks_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_shade_arena", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_simpleqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_subversion_strategy", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_subversion_strategy", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_multilingual", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_verified", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_sycophancy_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_sycophancy_eval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_synthetic_honeypots", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_synthetic_honeypots", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_terminal_bench_2_0", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_text_based_reinforcement_learning_task", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_text_based_reinforcement_learning_task", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_time_series_forecasting", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tool_use_prompt_injection_evaluation", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_tool_use_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vals_ai_finance_agent", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vals_ai_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vending_bench_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vending_bench_2", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vending_bench_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vending_bench_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vending_bench_arena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vending_bench_arena", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vmqa_vct", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vmqa_vct", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_webarena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_webarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_webarena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_webarena", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_webarena_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_webarena_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_webarena_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_webarena_verified", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wildchat", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_wildchat", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_xstest", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_xstest", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_arc_agi_2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_arc_agi_2", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 14, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_matharena_apex", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_matharena_apex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_simpleqa_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_terminal_bench_2_0", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2025", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_vending_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_video_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_ai_text_leaderboard", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_arena_ai_text_leaderboard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_arena_ai_text", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_arena_ai_text", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_gpqa_diamond", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 15, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_matharena_apex", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_matharena_apex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_mmmu_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_mmmu_pro", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_simpleqa_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_terminal_bench_2_0", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2025", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_vending_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_video_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_arc_agi_1", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp_long_context", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_browsecomp_long_context", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_factscore", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_factscore", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontiermath", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_hle", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_hmmt_february_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_longfact", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_longfact", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_mcp_atlas", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_chatgpt_factuality", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_openai_chatgpt_factuality", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_investment_banking_spreadsheet_tasks", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_openai_investment_banking_spreadsheet_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_mrcr", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_record_2_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_telecom", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_record_2_bench_telecom", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_screenspot_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver_benchmark_video_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_apex_agents", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_apex_agents", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_apex_agents", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_apex_agents", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_arc_agi_1", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_arc_agi_1", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_arc_agi_1", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_arc_agi_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_arc_agi_2", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_biglaw_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_biglaw_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_biglaw_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_biglaw_bench", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_browsecomp", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_browsecomp", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_browsecomp_long_context", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_browsecomp_long_context", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_factscore", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_factscore", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_finance_agent", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_financeagent", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_financeagent", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontier_science_research", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_frontier_science_research", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_frontier_science_research", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_frontier_science_research", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontiermath", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_frontiermath", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_frontiermath", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_gdpval", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_gdpval", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_gdpval", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_gpqa_diamond", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_hle", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_humanitys_last_exam", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_hmmt_february_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_longfact", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_longfact", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_mcp_atlas", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_scale_mcp_atlas", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_scale_mcp_atlas", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_mmmu_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_mmmu_pro", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_omnidocbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_omnidocbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_omnidocbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_omnidocbench", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_online_mind2web", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_online_mind2web", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_online_mind2web", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_online_mind2web", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_chatgpt_factuality", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_openai_chatgpt_factuality", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_investment_banking_spreadsheet_tasks", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_openai_investment_banking_spreadsheet_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_mrcr", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_openai_mrcr", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_openai_mrcr", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_osworld_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_osworld_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_osworld_verified", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_record_2_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_telecom", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_record_2_bench_telecom", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_tau2_bench_telecom", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_tau2_bench_telecom", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_screenspot_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_swe_bench_pro", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_toolathlon", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_toolathlon", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_video_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260305_introducing_gpt_5_4", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webarena_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260305_introducing_gpt_5_4_benchmark_webarena_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "occurrence_id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_webarena_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_webarena_verified", "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_apex_agents", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_apex_agents", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_arc_agi_1", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_biglaw_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_biglaw_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp_long_context", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_browsecomp_long_context", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_factscore", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_factscore", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_finance_agent", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontier_science_research", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_frontier_science_research", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontiermath", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_gpqa_diamond", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_graphwalks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_graphwalks", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_hle", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_humanitys_last_exam", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_hmmt_february_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_longfact", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_longfact", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_mcp_atlas", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_scale_mcp_atlas", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_scale_mcp_atlas", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_mmmu_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_mmmu_pro", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_omnidocbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_omnidocbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_omnidocbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_omnidocbench", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_online_mind2web", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_online_mind2web", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_chatgpt_factuality", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_openai_chatgpt_factuality", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_investment_banking_spreadsheet_tasks", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_openai_investment_banking_spreadsheet_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_mrcr", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_openai_mrcr", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_openai_mrcr", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_osworld_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_osworld_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_osworld_verified", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_record_2_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_telecom", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_record_2_bench_telecom", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_tau2_bench_telecom", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_tau2_bench_telecom", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_screenspot_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_swe_bench_pro", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_terminal_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_terminal_bench", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_toolathlon", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_toolathlon", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_video_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webarena_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260317_introducing_gpt_5_4_mini_and_nano_benchmark_webarena_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260319_qwen3_5_max_preview_benchmark_arena", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_text_general", "model_id": "model_qwen_model_qwen_20260319_qwen3_5_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260319_qwen3_5_max_preview_arena", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260319_qwen3_5_max_preview_arena", "release_qwen_20260319_qwen3_5_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-19", "release_id": "release_qwen_20260319_qwen3_5_max_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aa_lcr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_aa_lcr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_aa_lcr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_aa_lcr", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_ai2d_test", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ai2d_test", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ai2d_test", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_aime_2026", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_aime_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_av_speakerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_av_speakerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_av_speakerbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_av_speakerbench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_avut", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_avut", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_avut", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_avut", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_babyvision", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_babyvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bfcl_v4", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_bfcl_v4", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_bfcl_v4", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_bfcl_v4", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_browsecomp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp_zh", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_browsecomp_zh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_c_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_c_eval", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_cc_ocr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_cc_ocr", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_charxiv_rq", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_charxiv_rq", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_common_voice_15", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_common_voice_15", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_common_voice_15", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_common_voice_15", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_countbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_countbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_countbench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_crux_i", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_crux_i", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_crux_o", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_crux_o", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_dailyomni", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_dailyomni", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_dailyomni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_dailyomni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deep_planning", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_dynamath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_dynamath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_dynamath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_dynamath", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_embspatialbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_embspatialbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_embspatialbench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_erqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_erqa", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_fleurs", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_fleurs", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_fleurs", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_fleurs", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_global_piqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_global_piqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_gpqa", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hallusionbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_hle_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle_with_tools", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_november_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_hmmt_november_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_hmmt_november_2025", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hypersim", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_hypersim", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ifbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ifbench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ifeval", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_imoanswerbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_imoanswerbench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_kespeech", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_kespeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_kespeech", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_kespeech", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_korbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_korbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_librispeech", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_librispeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_librispeech", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_librispeech", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lingoqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_lingoqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_livecodebench_v6", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_longbench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_longbench_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_longbench_v2", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_lvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_lvbench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mathvision", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mathvista_mini", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mathvista_mini", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mathvista_mini", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mathvista_mini", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_maxife", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_maxife", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_mark", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_medxpertqa_mm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_medxpertqa_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_medxpertqa_mm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_medxpertqa_mm", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mir_1k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mir_1k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mir_1k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mir_1k", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mlvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mlvu", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmar", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmau", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmau", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmau", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmau", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmbench_en_1_1_dev", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmbench_en_1_1_dev", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mme_videoocr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mme_videoocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mme_videoocr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mme_videoocr", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlongbench_doc", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmlongbench_doc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlongbench_doc", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlongbench_doc", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlu_pro", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_prox", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlu_redux", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmmu", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmmu_pro", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmstar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmstar", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmsu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmsu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmsu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmsu", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mmvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmvu", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_multichallenge", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mvbench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_nova", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_nova", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_nuscene", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_nuscene", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ocrbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ocrbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ocrbench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_odinw13", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_odinw13", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_odinw13", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_omni_cloze", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_omni_cloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_omni_cloze", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_omni_cloze", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_omnidocbench_1_5", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_omnigaia", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_omnigaia", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_omnigaia", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_omnigaia", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_opencpop", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_opencpop", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_opencpop", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_opencpop", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_osworld_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_pmc_vqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_pmc_vqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_pmc_vqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_pmc_vqa", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_polymath", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_polymath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_qualcomminteractive", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_qualcomminteractive", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_qualcomminteractive", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_qualcomminteractive", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_realworldqa", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_refcoco", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_refcoco", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_refcoco", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_refspatialbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_refspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_rul_muchomusic", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_rul_muchomusic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_rul_muchomusic", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_rul_muchomusic", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_seal_0", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_seal_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_seccodebench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_seccodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_simplevqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_simplevqa", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_slake", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_slake", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_slake", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_slake", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_songformbench_chinese", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_songformbench_chinese", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_songformbench_chinese", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_songformbench_chinese", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_songformbench_harmonixset", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_songformbench_harmonixset", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_songformbench_harmonixset", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_songformbench_harmonixset", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_speechrole", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_speechrole", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_speechrole", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_speechrole", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_sunrgbd", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_sunrgbd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_supergpqa", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_agentless", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_swe_agentless", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tau2_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_tau2_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_tau2_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_tau2_bench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tool_decathlon", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_tool_decathlon", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_uro_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_uro_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_uro_bench_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_uro_bench_pro", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_v", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_v", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme_with_audio", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_videomme_with_audio", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_videomme_with_audio", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_videomme_with_audio", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_with_subtitles", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme_without_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_videomme_without_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_videomme_without_subtitles", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_videommmu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vita_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_voicebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_voicebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_voicebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_voicebench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_we_math", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_we_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_wenetspeech", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_wenetspeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_wenetspeech", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_wenetspeech", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_widesearch", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_wildspeech_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_wildspeech_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_wildspeech_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_wildspeech_bench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_wmt24", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260330_qwen3_5_omni", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_worldsense", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_worldsense", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_worldsense", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_worldsense", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_zerobench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_zerobench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_zerobench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_zerobench", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_zerobench_sub", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260330_qwen3_5_omni_benchmark_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_zerobench_sub", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_zerobench_sub", "release_qwen_20260330_qwen3_5_omni" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_agieval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_aime_2026", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_aime_2026", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_aime_2026", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_arc_c", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_big_bench_extra_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_big_bench_extra_hard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_bigbench_extra_hard", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_bigbench_extra_hard", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_bird_sql", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_coco_caption", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_coco_caption", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_codeforces", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_codeforces", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_codeforces", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_countbench_vqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_countbench_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_eclektic", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_eclektic", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_facts_grounding", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_facts_grounding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_flores", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_flores", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_global_mmlu_lite", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_global_mmlu_lite", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_gpqa_diamond", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_humanitys_last_exam", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_livecodebench_v6", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_livecodebench_v6", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_mbpp", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmlu_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmlu_pro", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_mmmlu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmmlu", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmmlu", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmmu_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmmu_pro", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural_questions", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_natural_questions", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ok_vqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_ok_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_perception_test_mcvqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_perception_test_mcvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_simpleqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_socialiqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_socialiqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_spatialsense_vqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_spatialsense_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tallyqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_tallyqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_winogrande", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_wmt24", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_xquad", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_benchmark_xquad", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aa_lcr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_aa_lcr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_aa_lcr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_aa_lcr", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d_test", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ai2d_test", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ai2d_test", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_aime_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_aime_2026", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_aime_2026", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_c_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_c_eval", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_cc_ocr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_cc_ocr", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_charxiv_rq", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_charxiv_rq", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_claweval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_claweval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_claweval", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_countbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_countbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_countbench", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deep_planning", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_deep_planning", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_deep_planning", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_dynamath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_dynamath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_dynamath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_dynamath", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_erqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_erqa", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_global_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_global_piqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_global_piqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_global_piqa", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_gpqa", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hle", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hle", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle_with_tools", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hle_with_tools", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hle_with_tools", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_february_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_february_2025", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_february_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_february_2026", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_february_2026", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_november_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_november_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_november_2025", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ifbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ifbench", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ifeval", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_imoanswerbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_imoanswerbench", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_include", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_include", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_livecodebench_v6", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_longbench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_longbench_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_longbench_v2", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mathvision", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_maxife", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_maxife", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_maxife", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_maxife", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mcp_atlas", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mcp_atlas", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mcp_atlas", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_mark", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mcp_mark", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mcp_mark", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mlvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mlvu", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlongbench_doc", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mmlongbench_doc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlongbench_doc", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlongbench_doc", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_pro", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_prox", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_prox", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_prox", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_redux", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmlu", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmu", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmu_pro", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmstar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmstar", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_nl2repo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_nl2repo", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nova", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_nova", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_nova", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_nova", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_odinw13", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_odinw13", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_odinw13", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_omnidocbench_1_5", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_omnidocbench_1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_omnidocbench_1_5", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_osworld_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_osworld_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_osworld_verified", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_polymath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_polymath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_polymath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_polymath", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenclawbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_qwenclawbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_qwenclawbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_qwenclawbench", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenwebbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_qwenwebbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_qwenwebbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_qwenwebbench", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_realworldqa", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refcoco", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_refcoco", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_refcoco", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_screenspot_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_screenspot_pro", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_simplevqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_simplevqa", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_skillsbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_skillsbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_skillsbench", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_supergpqa", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_multilingual", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_pro", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_verified", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau3_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_tau3_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tau3_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tau3_bench", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_terminal_bench_2_0", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tir_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_tir_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tir_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tir_bench", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tool_decathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_tool_decathlon", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tool_decathlon", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tool_decathlon", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_v", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_v", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_v_star", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_v_star", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_with_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videomme_with_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videomme_with_subtitles", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_without_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videomme_without_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videomme_without_subtitles", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videommmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videommmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videommmu", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vita_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_vita_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_vita_bench", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_we_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_we_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_we_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_we_math", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_widesearch", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_widesearch", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_widesearch", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260402_qwen3_6_benchmark_wmt24", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_wmt24_plus_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_wmt24_plus_plus", "release_qwen_20260402_qwen3_6" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_abc_bench_fragment_design", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_abc_bench_fragment_design", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_abc_bench_fragment", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_abc_bench_fragment", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_abc_bench_liquid_handling", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_abc_bench_liquid_handling", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_abc_bench_liquid", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_abc_bench_liquid", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agentdojo", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_agentdojo", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_agentdojo", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_agentdojo", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agentharm", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_agentharm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_agentharm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_agentharm", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agentic_cybermisuse", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_agentic_cybermisuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_agentic_cybermisuse", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_agentic_cybermisuse", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agentic_misalignment", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_agentic_misalignment", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_agentic_misalignment", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_agentic_misalignment", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_airs_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_airs_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_airs_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_airs_bench", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_alignment_faking", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_alignment_faking", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_alignment_faking", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_alignment_faking", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_arc_agi_2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_arc_agi_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_arc_agi_2", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biodesign_tools", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_biodesign_tools", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_biodesign_tools", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_biodesign_tools", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biotier_refusal", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_biotier_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_biotier_refusal", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_biotier_refusal", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_charxiv_reasoning", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_charxiv_reasoning", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chemical_agents_refusal", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_chemical_agents_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_chemical_agents_refusal", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_chemical_agents_refusal", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_cybench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_cybench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_cybench", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyber_benign_use", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_cyber_benign_use", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_cyber_benign_use", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_cyber_benign_use", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyber_misuse", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_cyber_misuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_cyber_misuse", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_cyber_misuse", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_cybergym", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_cybergym", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_cybergym", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyscenariobench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_cyscenariobench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_cyscenariobench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_cyscenariobench", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deceptionbench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_deceptionbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_deceptionbench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_deceptionbench", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deepsearchqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_deepsearchqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_deepsearchqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_deepsearchqa", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_erqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_erqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_erqa", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_fortress", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_fortress", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_fortress", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_fortress", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontierscience_research", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_frontierscience_research", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_frontier_science_research", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_frontier_science_research", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdm_stealth", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_gdm_stealth", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_gdm_stealth", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_gdm_stealth", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_gdpval_aa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_gdpval_aa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_gdpval_aa", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_gpqa_diamond", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_gpqa_diamond", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_grayswan_art", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_grayswan_art", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_grayswan_art", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_grayswan_art", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_healthbench_hard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_healthbench_hard", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_healthbench_hard", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_hle", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_humanitys_last_exam", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle_calibration", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_hle_calibration", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_hle_calibration", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_hle_calibration", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hpct", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_hpct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_hpct", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_hpct", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_iheval", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_iheval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_iheval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_iheval", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_impossiblebench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_impossiblebench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_impossiblebench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_impossiblebench", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_internal_sycophancy", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_internal_sycophancy", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_internal_sycophancy", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_internal_sycophancy", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_international_physics_olympiad_2025_theory", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_international_physics_olympiad_2025_theory", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_ipho_2025", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_ipho_2025", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_lab_bench_protocolqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_protocolqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_protocolqa", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_lab_bench_seqqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_seqqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_seqqa", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_livecodebench_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_livecodebench_pro", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_livecodebench_pro", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mask", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_mask", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_mask", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_mask", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mbct", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_mbct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_mbct", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_mbct", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_medxpertqa_multimodal", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_medxpertqa_multimodal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_medxpertqa_multimodal", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_medxpertqa_multimodal", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_medxpertqa_text", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_medxpertqa_text", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_medxpertqa_text", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_medxpertqa_text", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mle_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_mle_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_mle_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_mle_bench", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_mmmu_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_mmmu_pro", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_mmmu_pro", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_or_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_or_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_or_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_or_bench", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_prompted_evaluation_awareness", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_prompted_evaluation_awareness", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_prompted_evaluation_awareness", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_prompted_evaluation_awareness", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_screenspot_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_screenspot_pro", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_screenspot_pro", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_secure_code_generation", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_secure_code_generation", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_secure_code_generation", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_secure_code_generation", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_severe_cybermisuse_refusal", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_severe_cybermisuse_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_severe_cybermisuse_refusal", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_severe_cybermisuse_refusal", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_shade_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_shade_arena", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_shade_arena", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_shade_arena", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_simpleqa_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_simpleqa_verified", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_simpleqa_verified", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_simplevqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_simplevqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_simplevqa", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_social_engineering_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_social_engineering_evaluation", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_social_engineering_eval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_social_engineering_eval", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_social_engineering_refusal", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_social_engineering_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_social_engineering_refusal", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_social_engineering_refusal", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_strongreject_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_strongreject_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_strongreject_v2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_strongreject_v2", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_swe_bench_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_swe_bench_pro", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_swe_bench_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_swe_bench_verified", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau_bench_telecom", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_tau_bench_telecom", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_tau_bench_telecom", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_tau_bench_telecom", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_terminal_bench_2_0", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_terminal_bench_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_terminal_bench_2", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vct", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_vct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_vct", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_vct", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmdp_bio", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_wmdp_bio", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_wmdp_bio", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_wmdp_bio", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmdp_chem", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_wmdp_chem", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_wmdp_chem", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_wmdp_chem", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmdp_cyber", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_wmdp_cyber", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_wmdp_cyber", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_wmdp_cyber", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_zerobench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260408_introducing_muse_spark_benchmark_zerobench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_zerobench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_zerobench", "release_meta_20260408_introducing_muse_spark" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ai2d_test", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_ai2d_test", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_ai2d_test", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aime_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_aime_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_aime_2026", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_aime_2026", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_c_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_c_eval", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_cc_ocr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_cc_ocr", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_charxiv_rq", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_charxiv_rq", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_claweval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_claweval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_claweval", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deep_planning", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_deep_planning", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_deep_planning", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_embspatialbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_embspatialbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_embspatialbench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_gpqa", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hallusionbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hallusionbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hallusionbench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hle", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hle", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_february_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_february_2025", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_february_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_february_2026", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_february_2026", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_november_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_november_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_november_2025", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_imoanswerbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_imoanswerbench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_livecodebench_v6", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_lvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_lvbench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvista_mini", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_mathvista_mini", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mathvista_mini", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mathvista_mini", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_mcp_atlas", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mcp_atlas", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mcp_atlas", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_mark", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mcp_mark", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mcp_mark", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mlvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mlvu", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmbench_en_1_1_dev", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmbench_en_1_1_dev", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmlu_pro", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmlu_redux", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmmu", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmmu_pro", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mvbench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_nl2repo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_nl2repo", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_odinw13", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_odinw13", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_odinw13", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_omnidocbench_1_5", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_omnidocbench_1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_omnidocbench_1_5", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenclawbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_qwenclawbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_qwenclawbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_qwenclawbench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenwebbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_qwenwebbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_qwenwebbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_qwenwebbench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_realworldqa", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refcoco", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_refcoco", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_refcoco", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refspatialbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_refspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_refspatialbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_refspatialbench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_simplevqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_simplevqa", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_skillsbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_skillsbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_skillsbench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_supergpqa", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_multilingual", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_pro", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_verified", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau3_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_tau3_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_tau3_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_tau3_bench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_terminal_bench_2_0", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tool_decathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_tool_decathlon", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_tool_decathlon", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_tool_decathlon", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_with_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videomme_with_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videomme_with_subtitles", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_without_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videomme_without_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videomme_without_subtitles", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videommmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videommmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videommmu", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vita_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_vita_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_vita_bench", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_widesearch", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_widesearch", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_widesearch", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_zerobench_sub", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260415_qwen3_6_35b_a3b_benchmark_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_zerobench_sub", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_zerobench_sub", "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_aider_polyglot", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_aime_2025", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_arc_agi_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_arc_agi_1", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_arc_agi_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_arc_agi_2", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_biglaw_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_biglaw_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_biglaw_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_biglaw_bench", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_biomysterybench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_biopipelinebench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_biopipelinebench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_browsecomp", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_browsecomp", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_browsecomp_plus", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_browsecomp_plus", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_charxiv_reasoning", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cloningscenarios", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_creative_biology", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cursorbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cursorbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cursorbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cursorbench", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cybench", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cybergym", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cybergym", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deepsearchqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_deepsearchqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_deepsearchqa", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_draco", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_draco", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_draco", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_draco", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_figqa", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_finance_agent", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_finance_agent", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_firefox_147_exploitation_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_firefox_147_exploitation_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_firefox_147_exploitation_evaluation", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_firefox_147_exploitation_evaluation", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_gdpval_aa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_gdpval_aa", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_gpqa_diamond", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_graphwalks", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_graphwalks", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_hle", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_humanity_s_last_exam", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_kernel_optimization", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_kernel_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_figqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_lab_bench_figqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_lab_bench_figqa", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_lab_bench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_lab_bench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_llm_training_optimization", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_llm_training_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_1", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_2", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mcp_atlas", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mcp_atlas", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mmmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mmmlu", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mmmu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mmmu_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_novel_compiler", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_officeqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_officeqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_officeqa", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_officeqa_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_officeqa_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_officeqa_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_officeqa_pro", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_mrcr_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_openai_mrcr_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_openai_mrcr_v2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_openai_mrcr_v2", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_openrca", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_openrca", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_osworld", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_osworld", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_quadruped_locomotion", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_quadruped_locomotion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_rakuten_swe_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_rakuten_swe_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_rakuten_swe_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_rakuten_swe_bench", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_real_world_finance", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_real_world_finance", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_airline_corrected", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_record_2_bench_airline_corrected", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_airline_original", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_record_2_bench_airline_original", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_record_2_bench_retail", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_telecom", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_record_2_bench_telecom", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_screenspot_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_screenspot_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_screenspot_pro", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_spreadsheetbench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_spreadsheetbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_multilingual", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_multimodal", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_multimodal", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_multimodal", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_pro", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_verified", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_synthesis_screening_evasion", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_terminal_bench_2_0", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_text_based_reinforcement_learning", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_text_based_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_time_series_forecasting", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_usamo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_usamo_2026", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_usamo_2026", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_vending_bench_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_vending_bench_2", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_webarena", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260416_introducing_claude_opus_4_7_benchmark_webarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260416_introducing_claude_opus_4_7", "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bixbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_benchmark_bixbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "model_id": "model_openai_model_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "occurrence_id": "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_bixbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_bixbench", "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cloningqa", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_benchmark_cloningqa", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "model_id": "model_openai_model_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "occurrence_id": "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_cloningqa", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_cloningqa", "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_labbench2", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_benchmark_labbench2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "model_id": "model_openai_model_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "occurrence_id": "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_labbench2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_labbench2", "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aa_lcr", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_aa_lcr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d_test", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2026", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_aime_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_claweval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_countbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_deep_planning", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_dynamath", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_dynamath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_global_piqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_global_piqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle_with_tools", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_february_2026", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_november_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_longbench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_maxife", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_maxife", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mcp_atlas", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_mark", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlongbench_doc", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mmlongbench_doc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_prox", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_nl2repo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_nl2repo", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_nova", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_nova", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_odinw13", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_omnidocbench_1_5", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_osworld_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_polymath", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_polymath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_qwenchinesebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_qwenchinesebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenchinesebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenchinesebench", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenclawbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_qwenclawbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenclawbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenclawbench", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_qwenwebbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_qwenwebbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenwebbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenwebbench", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_refcoco", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_scicode", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_scicode", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_scicode", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_scicode", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_skillsbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_skillsbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_skillsbench", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_supergpqa", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_swe_bench_pro", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tau3_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_tau3_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_terminal_bench_2_0", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_tir_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_tir_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_tool_decathlon", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_tool_decathlon", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolcallformatifbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_toolcallformatifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_toolcallformatifbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_toolcallformatifbench", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_v", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_v", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme_with_subtitles", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme_without_subtitles", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_videommmu", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vita_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_we_math", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_we_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_widesearch", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260418_qwen3_6_max_preview_benchmark_wmt24", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260418_qwen3_6_max_preview", "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d_test", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_aime_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_aime_2026", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_aime_2026", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_androidworld", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_androidworld", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_c_eval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_c_eval", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cc_ocr", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_cc_ocr", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_cc_ocr", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_charxiv_rq", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_charxiv_rq", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_claweval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_claweval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_claweval", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_countbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_countbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_countbench", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deep_planning", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_dynamath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_dynamath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_dynamath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_dynamath", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_embspatialbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_embspatialbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_embspatialbench", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_erqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_erqa", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_gpqa_diamond", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hallusionbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hle", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hle", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_february_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_february_2025", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hmmt_february_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_february_2026", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_february_2026", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hmmt_november_2025", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_november_2025", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_november_2025", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_imoanswerbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_imoanswerbench", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_livecodebench_v6", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista_mini", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mathvista_mini", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mathvista_mini", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mathvista_mini", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mcp_atlas", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_mark", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mlvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mlvu", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmbench_en_1_1_dev", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmbench_en_1_1_dev", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmlu_pro", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmlu_redux", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmmu", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmmu_pro", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmstar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmstar", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mvbench", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_nl2repo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_nl2repo", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_ocrbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_ocrbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_ocrbench", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_odinw13", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_omnidocbench_1_5", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_qwenclawbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_qwenclawbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_qwenclawbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_qwenclawbench", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_qwenwebbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_qwenwebbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_qwenwebbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_qwenwebbench", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_realworldqa", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_refcoco", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_refcoco", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_refcoco", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_refspatialbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_refspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_refspatialbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_refspatialbench", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_simplevqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_simplevqa", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_skillsbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_skillsbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_skillsbench", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_supergpqa", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_multilingual", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_pro", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_verified", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_tau3_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_tau3_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_terminal_bench_2_0", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_tool_decathlon", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_tool_decathlon", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_v", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_v", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_v_star", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_v_star", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_with_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_videomme_with_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_videomme_with_subtitles", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_videomme_without_subtitles", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_videommmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_videommmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_videommmu", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_vita_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vlmsareblind", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_vlmsareblind", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_vlmsareblind", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_vlmsareblind", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_widesearch", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_zerobench_sub", "consecutive_reviewed_omissions": 1, "id": "status_release_qwen_20260422_qwen3_6_27b_benchmark_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_qwen_20260422_qwen3_6_27b", "release_qwen_20260422_qwen3_6_27b" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 6, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_apex_agents", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_apex_agents", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_arc_agi_1", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_arc_agi_1", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_arc_agi_1", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_biglaw_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_biglaw_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bixbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_bixbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_bixbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_bixbench", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_browsecomp", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_browsecomp", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_browsecomp_long_context", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_browsecomp_long_context", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_capture_the_flag_challenges", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_capture_the_flag_challenges", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_capture_the_flag_challenges", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_capture_the_flag_challenges", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cot_control", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_cot_control", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_cot_control", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_cot_control", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_cybergym", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_cybergym", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_cybergym", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_expert_swe", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_expert_swe", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_expert_swe", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_expert_swe", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_factscore", "consecutive_reviewed_omissions": 6, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_factscore", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_finance_agent", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_financeagent", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_financeagent", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_frontier_science_research", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_frontier_science_research", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontiermath", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_frontiermath", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_frontiermath", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_gdpval", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_gdpval", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_gdpval", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_genebench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_genebench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_genebench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_genebench", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_gpqa_diamond", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_graphwalks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_graphwalks", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_healthbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_healthbench", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_healthbench_professional", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_healthbench_professional", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_healthbench_professional", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_healthbench_professional", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_hle", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_humanitys_last_exam", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_hmmt_february_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_longfact", "consecutive_reviewed_omissions": 6, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_longfact", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_mcp_atlas", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 6, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_mmmu_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_mmmu_pro", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 6, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_nucleobench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_nucleobench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_nucleobench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_nucleobench", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_officeqa_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_officeqa_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_officeqa_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_officeqa_pro", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_omnidocbench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_omnidocbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_online_mind2web", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_online_mind2web", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_chatgpt_factuality", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_openai_chatgpt_factuality", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_investment_banking_spreadsheet_tasks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_openai_investment_banking_spreadsheet_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_openai_investment_banking_spreadsheet_tasks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_openai_investment_banking_spreadsheet_tasks", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_openai_mrcr", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_openai_mrcr", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_openai_mrcr", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_osworld_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_osworld_verified", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_osworld_verified", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_record_2_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_telecom", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_record_2_bench_telecom", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_tau2_bench_telecom", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_tau2_bench_telecom", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_screenspot_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_swe_bench_pro", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_terminal_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_terminal_bench", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_toolathlon", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_toolathlon", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 4, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_video_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webarena_verified", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260423_introducing_gpt_5_5_benchmark_webarena_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260423_introducing_gpt_5_5", "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_agieval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_agieval", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_apex", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_apex", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_apex", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_apex", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_apex_shortlist", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_apex_shortlist", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_apex_shortlist", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_apex_shortlist", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_bbh", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_bbh", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bigcodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_bigcodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_bigcodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_bigcodebench", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_browsecomp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_browsecomp", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_c_eval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_c_eval", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chinese_simpleqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_chinese_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_chinese_simpleqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_chinese_simpleqa", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_cluewsc", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_cluewsc", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_cmath", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_cmath", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_cmmlu", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_cmmlu", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_codeforces", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_codeforces", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_corpusqa_1m", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_corpusqa_1m", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_corpusqa_1m", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_corpusqa_1m", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_drop", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_drop", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_facts_parametric", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_facts_parametric", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_facts_parametric", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_facts_parametric", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_gdpval_aa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_gdpval_aa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_gdpval_aa", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_gpqa_diamond", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_gsm8k", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_gsm8k", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_hellaswag", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_hellaswag", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_hle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_humanity_s_last_exam", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_2026_feb", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_hmmt_2026_feb", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_hmmt_2026_feb", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_hmmt_2026_feb", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_humaneval", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_humaneval", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_imoanswerbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_imoanswerbench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_imoanswerbench", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_livecodebench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_livecodebench", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_longbench_v2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_longbench_v2", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_math", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_math", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcpatlas", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_mcpatlas", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_mcpatlas", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mcpatlas", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcpatlas_public", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_mcpatlas_public", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_mcpatlas_public", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mcpatlas_public", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_mgsm", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mgsm", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu_pro", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu_pro", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu_redux", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu_redux", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_mmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_mmmlu", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mmmlu", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mrcr_1m", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_mrcr_1m", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_mrcr_1m", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mrcr_1m", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multiloko", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_multiloko", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_multiloko", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_multiloko", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_simpleqa_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_simpleqa_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_simpleqa_verified", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_supergpqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_supergpqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_supergpqa", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_swe_bench_multilingual", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_swe_bench_verified", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_swe_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_swe_pro", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_swe_pro", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_terminal_bench_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_terminal_bench_2_0", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_toolathlon", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_toolathlon", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_toolathlon", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_triviaqa", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_triviaqa", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260424_deepseek_v4_benchmark_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_family", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_family_winogrande", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_winogrande", "release_deepseek_20260424_deepseek_v4" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 7, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_apex_agents", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_apex_agents", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_arc_agi_1", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_biglaw_bench", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_biglaw_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bixbench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_bixbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp_long_context", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_browsecomp_long_context", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_capture_the_flag_challenges", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_capture_the_flag_challenges", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 6, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cot_control", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_cot_control", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_cybergym", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_expert_swe", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_expert_swe", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_factscore", "consecutive_reviewed_omissions": 7, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_factscore", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_finance_agent", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontier_science_research", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_frontier_science_research", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontiermath", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_genebench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_genebench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_healthbench_professional", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_healthbench_professional", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_hle", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_february_2025", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_hmmt_february_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_longfact", "consecutive_reviewed_omissions": 7, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_longfact", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_mcp_atlas", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 7, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multichallenge", "consecutive_reviewed_omissions": 7, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_nucleobench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_nucleobench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_officeqa_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_officeqa_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_omnidocbench", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_omnidocbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_online_mind2web", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_online_mind2web", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_chatgpt_factuality", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_openai_chatgpt_factuality", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_investment_banking_spreadsheet_tasks", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_openai_investment_banking_spreadsheet_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_mrcr", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_osworld_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_record_2_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_telecom", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_record_2_bench_telecom", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_screenspot_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 5, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_video_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webarena_verified", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize_benchmark_webarena_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "release_id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_ai_text_leaderboard", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_arena_ai_text_leaderboard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_charxiv_reasoning", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_charxiv_reasoning", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_gdpval_aa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_gdpval_aa", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_gdpval_aa", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 16, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_matharena_apex", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_matharena_apex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_mcp_atlas", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_mcp_atlas", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_mcp_atlas", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 11, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_simpleqa_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_terminal_bench_2_0", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_terminal_bench_2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_terminal_bench_2", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2025", "consecutive_reviewed_omissions": 7, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_vending_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_video_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_apex", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_apex", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_apex", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_apex", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bfcl_v4", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_bfcl_v4", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_bfcl_v4", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_bfcl_v4", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_claweval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_claweval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_claweval", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_coworkbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_coworkbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_coworkbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_coworkbench", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_critpt", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_critpt", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_critpt", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_critpt", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_global_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_global_piqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_global_piqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_global_piqa", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_gpqa_diamond", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hle", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hle", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle_with_tools", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hle_with_tools", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hle_with_tools", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_february_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hmmt_february_2026", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hmmt_february_2026", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_ifbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_ifbench", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_ifeval", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_imoanswerbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_imoanswerbench", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_include", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_include", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_kernel_bench_l3", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_kernel_bench_l3", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_kernel_bench_l3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_kernel_bench_l3", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_livecodebench", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_maxife", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_maxife", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_maxife", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_maxife", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_mcp_atlas", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mcp_atlas", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mcp_atlas", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_mark", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mcp_mark", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mcp_mark", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_pro", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_prox", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_prox", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_prox", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_redux", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmmlu", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mrcr_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_mrcr_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mrcr_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mrcr_v2", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_nl2repo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_nl2repo", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nova", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_nova", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_nova", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_nova", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_polymath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_polymath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_polymath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_polymath", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenclaw", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_qwenclaw", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenclaw", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenclaw", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwensvg", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_qwensvg", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwensvg", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwensvg", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenwebdev", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_qwenwebdev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenwebdev", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenwebdev", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenworldbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_qwenworldbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenworldbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenworldbench", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_scicode", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_scicode", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_scicode", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_scicode", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_skillsbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_skillsbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_skillsbench", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_spreadsheetbench_v1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_spreadsheetbench_v1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_spreadsheetbench_v1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_spreadsheetbench_v1", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_supergpqa", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_multilingual", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_pro", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_verified", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_terminal_bench_2_0", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vita_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_vita_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_vita_bench", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260520_qwen3_7_benchmark_wmt24", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_wmt24_plus_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_wmt24_plus_plus", "release_qwen_20260520_qwen3_7" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aav_capsid_packaging_prediction", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_aav_capsid_packaging_prediction", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_aav_capsid_packaging_prediction", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_aav_capsid_packaging_prediction", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_aider_polyglot", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_aime_2025", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arxivmath", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_arxivmath", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_arxivmath", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_arxivmath", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_automation_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_automationbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_automationbench", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_bias_benchmark_for_question_answering", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_bias_benchmark_for_question_answering", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_biglaw_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biglaw_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_biomysterybench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_biomysterybench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biomysterybench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biomysterybench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biomysterybench_verified", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biopipelinebench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biopipelinebench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_biopipelinebench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biopipelinebench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biopipelinebench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biopipelinebench_verified", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_black_box_rna_sequence_modeling_and_design", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_black_box_rna_sequence_modeling_and_design", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_black_box_rna_sequence_modeling_and_design", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_black_box_rna_sequence_modeling_and_design", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_browsecomp", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_browsecomp", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_browsecomp_plus", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_browsecomp_plus", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartmuseum", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_chartmuseum", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_chartmuseum", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_chartmuseum", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqapro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_chartqapro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_chartqapro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_chartqapro", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_charxiv_reasoning", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cloningscenarios", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_creative_biology", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cursorbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cursorbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cursorbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cursorbench", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cybergym", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cybergym", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deep_research_accuracy_completeness_and_objectivity", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_deep_research_accuracy_completeness_and_objectivity", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_deep_research_accuracy_completeness_and_objectivity", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_deep_research_accuracy_completeness_and_objectivity", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deepsearchqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_deepsearchqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_deepsearchqa", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_draco", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_draco", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_exploitbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_exploitbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_exploitbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_exploitbench", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_figqa", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_finance_agent", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_finance_agent", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_firefox_147_exploitation_evaluation", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_firefox_147_exploitation_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_firefox_exploit_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_firefox_exploit_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_firefox_exploit_evaluation", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_firefox_exploit_evaluation", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontierswe", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_frontierswe", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_frontierswe", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_frontierswe", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_gdpval_aa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_gdpval_aa", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_global_mmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_global_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_global_mmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_global_mmlu", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_gpqa_diamond", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_graphwalks_bfs_256k", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks_bfs_256k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks_bfs_256k", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks_bfs_256k", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_graphwalks_parents_256k", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks_parents_256k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks_parents_256k", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks_parents_256k", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench_professional", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_healthbench_professional", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_healthbench_professional", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_healthbench_professional", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_hle", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_humanity_s_last_exam", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_include", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_include", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_include", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_kernel_optimization", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_kernel_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_figqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_lab_bench_figqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_lab_bench_figqa", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_lab_bench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_lab_bench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_labbench2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_labbench2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_labbench2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_labbench2", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_latchbio_bioinformatics", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_latchbio_bioinformatics", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_latchbio_bioinformatics", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_latchbio_bioinformatics", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_legal_agent_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_legal_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_legal_agent_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_legal_agent_benchmark", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_llm_training_optimization", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_llm_training_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_1", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_2", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mask", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_mask", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_model_alignment_between_statements_and_knowledge", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_model_alignment_between_statements_and_knowledge", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_mcp_atlas", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_mcp_atlas", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_minimal_linuxbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_minimal_linuxbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_minimal_linuxbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_minimal_linuxbench", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_mmmu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_mmmu_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_task_indic_language_understanding_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_multi_task_indic_language_understanding_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_multi_task_indic_language_understanding_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_multi_task_indic_language_understanding_benchmark", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_novel_compiler", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_officeqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_officeqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_officeqa", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_officeqa_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_officeqa_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_online_mind2web", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_online_mind2web", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_online_mind2web", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_online_mind2web", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_mrcr_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_openai_mrcr_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openrca", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_openrca", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_oss_fuzz", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_oss_fuzz", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_oss_fuzz", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_oss_fuzz", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_osworld_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_osworld_verified", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_programbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_programbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_programbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_programbench", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_proteingym_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_proteingym_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_proteingym_hard", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_proteingym_hard", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quadruped_locomotion", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_quadruped_locomotion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_rakuten_swe_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_rakuten_swe_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_real_world_finance", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_real_world_finance", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_airline_corrected", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_record_2_bench_airline_corrected", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_airline_original", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_record_2_bench_airline_original", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_record_2_bench_retail", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_telecom", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_record_2_bench_telecom", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_screenspot_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_screenspot_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_screenspot_pro", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_securebio_dna_synthesis_screening_evasion", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_securebio_dna_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_securebio_dna_synthesis_screening_evasion", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_securebio_dna_synthesis_screening_evasion", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_securebio_virology_capability_test", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_securebio_virology_capability_test", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_securebio_virology_capability_test", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_securebio_virology_capability_test", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_shade_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_shade_arena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_shade_arena", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_spreadsheetbench", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_spreadsheetbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_super_agent_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_super_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_super_agent_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_super_agent_benchmark", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_multilingual", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_multimodal", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_pro", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_verified", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_synthesis_screening_evasion", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_terminal_bench_2_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_terminal_bench_2_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_terminal_bench_2_1", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_text_based_reinforcement_learning", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_text_based_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_time_series_forecasting", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_toolathlon", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_toolathlon", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_toolathlon", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_usamo_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_usamo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_usamo_2026", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_usamo_2026", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_vending_bench_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_vending_bench_2", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_webarena", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260528_introducing_claude_opus_4_8_benchmark_webarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260528_introducing_claude_opus_4_8", "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_androidworld", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_androidworld", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_apex", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_apex", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_apex", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_apex", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_babyvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_babyvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_babyvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_babyvision", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bc_vl", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_bc_vl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_bc_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_bc_vl", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bfcl_v4", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_bfcl_v4", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_bfcl_v4", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_bfcl_v4", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_charxiv_rq", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_charxiv_rq", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_claweval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_claweval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_claweval", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_claweval_mm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_claweval_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_claweval_mm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_claweval_mm", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_countqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_countqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_countqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_countqa", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_coworkbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_coworkbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_coworkbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_coworkbench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_critpt", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_critpt", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_critpt", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_critpt", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deep_planning", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_deep_planning", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_deep_planning", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ego3d_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_ego3d_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ego3d_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ego3d_bench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_erqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_erqa", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_global_piqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_global_piqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_global_piqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_global_piqa", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_gpqa_diamond", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hipho", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_hipho", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hipho", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hipho", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hle", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hle", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hmmt_february_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hmmt_february_2026", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hmmt_february_2026", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ifbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ifbench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifeval", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ifeval", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ifeval", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_imoanswerbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_imoanswerbench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_include", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_include", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_kernel_bench_l3", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_kernel_bench_l3", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_kernel_bench_l3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_kernel_bench_l3", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lingoqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_lingoqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_lingoqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_lingoqa", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_livecodebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_livecodebench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_lvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_lvbench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mathvision", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_maxife", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_maxife", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_maxife", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_maxife", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mcp_atlas", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mcp_atlas", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mcp_atlas", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_mark", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mcp_mark", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mcp_mark", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_medxpertqa_mm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_medxpertqa_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_medxpertqa_mm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_medxpertqa_mm", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mlvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mlvu", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmbc", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mmbc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmbc", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmbc", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_pro", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_prox", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_prox", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_prox", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_redux", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmmlu", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmmu_pro", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmsearchplus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mmsearchplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmsearchplus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmsearchplus", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mrcr_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_mrcr_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mrcr_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mrcr_v2", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_nl2repo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_nl2repo", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nova", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_nova", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_nova", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_nova", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocr_bench_v2_chinese", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_ocr_bench_v2_chinese", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ocr_bench_v2_chinese", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ocr_bench_v2_chinese", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocr_bench_v2_english", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_ocr_bench_v2_english", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ocr_bench_v2_english", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ocr_bench_v2_english", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_odinw13", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_odinw13", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_odinw13", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_omnidocbench_1_5", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_omnidocbench_1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_omnidocbench_1_5", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_osworld_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_osworld_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_osworld_verified", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_polymath", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_polymath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_polymath", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_polymath", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenclaw", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_qwenclaw", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenclaw", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenclaw", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwensvg", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_qwensvg", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwensvg", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwensvg", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenvision2code", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_qwenvision2code", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenvision2code", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenvision2code", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenwebdev", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_qwenwebdev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenwebdev", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenwebdev", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenworldbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_qwenworldbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenworldbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenworldbench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_realworldqa", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_scicode", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_scicode", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_scicode", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_scicode", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_screenspot_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_screenspot_pro", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_simplevqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_simplevqa", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_skillsbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_skillsbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_skillsbench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_spreadsheetbench_v1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_spreadsheetbench_v1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_spreadsheetbench_v1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_spreadsheetbench_v1", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_supergpqa", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_surds", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_surds", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_surds", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_surds", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_multilingual", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_pro", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_verified", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_terminal_bench_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_terminal_bench_2_0", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_tvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_tvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_tvbench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_with_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_videomme_with_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_videomme_with_subtitles", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videommmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_videommmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_videommmu", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_visfactor", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_visfactor", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_visfactor", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_visfactor", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vita_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_vita_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_vita_bench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vladbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_vladbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_vladbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_vladbench", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_wmt24", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_wmt24_plus_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_wmt24_plus_plus", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_worldvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260601_qwen3_7_plus_benchmark_worldvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_worldvqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_worldvqa", "release_qwen_20260601_qwen3_7_plus" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_agieval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2026", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_aime_2026", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_c", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_arc_c", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_easy", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_big_bench_extra_hard", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_big_bench_extra_hard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bird_sql", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_bird_sql", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_boolq", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_coco_caption", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_coco_caption", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_codeforces", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_countbench_vqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_countbench_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_eclektic", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_eclektic", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_facts_grounding", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_facts_grounding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_flores", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_flores", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_global_mmlu_lite", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_global_mmlu_lite", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_livecodebench_v6", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mbpp", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_mbpp", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_mmmlu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural_questions", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_natural_questions", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ok_vqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_ok_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_perception_test_mcvqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_perception_test_mcvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_piqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_simpleqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_socialiqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_socialiqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_spatialsense_vqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_spatialsense_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tallyqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_tallyqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_winogrande", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt24", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_wmt24", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_xquad", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi_benchmark_xquad", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bixbench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_benchmark_bixbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cloningqa", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_benchmark_cloningqa", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_genebench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_benchmark_genebench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "occurrence_id": "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_genebench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_genebench", "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_labbench2", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_benchmark_labbench2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_labworkbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_benchmark_labworkbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "occurrence_id": "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_labworkbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_labworkbench", "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lifescibench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_benchmark_lifescibench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "occurrence_id": "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_lifescibench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_lifescibench", "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_medchembench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_benchmark_medchembench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "occurrence_id": "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_medchembench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_medchembench", "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arxivmath", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_arxivmath", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_arxivmath", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_arxivmath", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_automation_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_automationbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_automationbench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_bias_benchmark_for_question_answering", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_bias_benchmark_for_question_answering", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_benchcad", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_benchcad", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_benchcad", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_benchcad", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biomysterybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_biomysterybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_biomysterybench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_blueprint_bench_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_blueprint_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_blueprint_bench_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_blueprint_bench_2", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_browsecomp", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_browsecomp", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartmuseum", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_chartmuseum", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_chartmuseum", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_chartmuseum", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqapro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_chartqapro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_chartqapro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_chartqapro", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_charxiv_reasoning", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_critpt", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_critpt", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_critpt", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_critpt", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cursorbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cursorbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cursorbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cursorbench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cybergym", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cybergym", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyscenariobench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cyscenariobench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cyscenariobench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cyscenariobench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deep_research_accuracy_completeness_and_objectivity", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_deep_research_accuracy_completeness_and_objectivity", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_deep_research_accuracy_completeness_and_objectivity", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_deep_research_accuracy_completeness_and_objectivity", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deepsearchqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_deepsearchqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_deepsearchqa", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_exploitbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_exploitbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_exploitbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_exploitbench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_finance_agent", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_finance_agent", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_firefox_147_exploit_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_firefox_147_exploit_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_firefox_147_exploit_evaluation", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_firefox_147_exploit_evaluation", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontierbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontierbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontierbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontierbench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontiercode", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontiercode", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontiercode", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontiercode", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontierswe", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontierswe", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontierswe", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontierswe", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdp_pdf", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gdp_pdf", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gdp_pdf", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gdp_pdf", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gdpval_aa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gdpval_aa", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_global_mmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_global_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_global_mmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_global_mmlu", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gpqa_diamond", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_graphwalks", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_graphwalks", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthadminbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthadminbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthadminbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthadminbench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthbench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench_professional", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthbench_professional", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthbench_professional", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthbench_professional", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hebbia_finance_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_hebbia_finance_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_hebbia_finance_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_hebbia_finance_benchmark", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_hle", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_humanity_s_last_exam", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_include", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_include", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_include", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_figqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_lab_bench_figqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_lab_bench_figqa", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_labbench2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_labbench2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_labbench2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_labbench2", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_latchbio_bioinformatics", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_latchbio_bioinformatics", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_latchbio_bioinformatics", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_latchbio_bioinformatics", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_legal_agent_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_legal_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_legal_agent_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_legal_agent_benchmark", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mask", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_mask", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_model_alignment_between_statements_and_knowledge", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_model_alignment_between_statements_and_knowledge", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_mcp_atlas", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_mcp_atlas", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_minimal_linuxbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_minimal_linuxbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_minimal_linuxbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_minimal_linuxbench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multi_task_indic_language_understanding_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_multi_task_indic_language_understanding_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_multi_task_indic_language_understanding_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_multi_task_indic_language_understanding_benchmark", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_officeqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_officeqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_officeqa", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_oss_fuzz", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_oss_fuzz", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_oss_fuzz", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_oss_fuzz", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_osworld_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_osworld_verified", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_programbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_programbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_programbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_programbench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_proteingym_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_proteingym_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_proteingym_hard", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_proteingym_hard", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_real_world_finance_v1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_real_world_finance_v1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_real_world_finance_v1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_real_world_finance_v1", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_real_world_finance_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_real_world_finance_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_real_world_finance_v2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_real_world_finance_v2", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_riemannbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_riemannbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_riemannbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_riemannbench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_screenspot_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_screenspot_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_screenspot_pro", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_shade_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_shade_arena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_shade_arena", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_multilingual", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multimodal", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_multimodal", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_multimodal", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_pro", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_verified", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench_2_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_terminal_bench_2_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_terminal_bench_2_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_terminal_bench_2_1", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_toolathlon", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_toolathlon", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_toolathlon", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_usamo_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_usamo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_usamo_2026", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_usamo_2026", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vending_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_vending_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_vending_bench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_vending_bench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vibench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_vibench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_vibench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_vibench", "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash_benchmark_androidworld", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_computer_use", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-24", "release_id": "release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_online_mind2web", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash_benchmark_online_mind2web", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_computer_use", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-24", "release_id": "release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_webvoyager", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash_benchmark_webvoyager", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_computer_use", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-24", "release_id": "release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_exploitbench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_benchmark_exploitbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "occurrence_id": "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_exploitbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_exploitbench", "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "release_id": "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_exploitgym", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_benchmark_exploitgym", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "occurrence_id": "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_exploitgym", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_exploitgym", "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "release_id": "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_genebench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_benchmark_genebench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "occurrence_id": "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_genebench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_genebench", "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "release_id": "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "occurrence_id": "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_terminal_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_terminal_bench", "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "release_id": "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aa_briefcase", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_aa_briefcase", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_aa_briefcase", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_aa_briefcase", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agent_red_teaming", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_agent_red_teaming", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_agentic_misalignment_suite", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_agentic_misalignment_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_ai2d", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_aime_2025", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arxivmath", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_arxivmath", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_arxivmath", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_arxivmath", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_automation_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_automationbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_automationbench", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_bbh", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_bias_benchmark_for_question_answering", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_bias_benchmark_for_question_answering", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_benchcad", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_benchcad", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_benchcad", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_benchcad", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biomysterybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_biomysterybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_biomysterybench", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_biopipelinebench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_biopipelinebench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_blackmail_suite", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_blackmail_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_browsecomp", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_browsecomp", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_chartmuseum", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_chartmuseum", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_chartmuseum", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_chartmuseum", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_chartqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_charxiv_reasoning", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cloningscenarios", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_computer_use_prompt_injection_evaluation", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_computer_use_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_creative_biology", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cursorbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cursorbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cursorbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cursorbench", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cybergym", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cybergym", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deepsearchqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_dna_synthesis_screening_evasion", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_dna_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_docvqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_drop", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_exploitbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_exploitbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_exploitbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_exploitbench", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_false_premise_questions", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_false_premise_questions", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_figqa", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_financial_crimes", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_financial_crimes", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_financial_services_benchmark", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_financial_services_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_firefox_147_exploit_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_firefox_147_exploit_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_firefox_147_exploit_evaluation", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_firefox_147_exploit_evaluation", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontiercode_v1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_frontiercode_v1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_frontiercode_v1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_frontiercode_v1", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdp_pdf", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdp_pdf", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdp_pdf", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdp_pdf", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval_aa_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdpval_aa_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdpval_aa_v2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdpval_aa_v2", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_global_mmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_global_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_global_mmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_global_mmlu", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gsm8k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_healthbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_healthbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_healthbench", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench_professional", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_healthbench_professional", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_healthbench_professional", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_healthbench_professional", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_hle", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_humanity_s_last_exam", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_humaneval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_impossible_tasks", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_impossible_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_incalmo_cyber_ranges", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_incalmo_cyber_ranges", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_include", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_include", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_include", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_2", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_internal_ai_research_evaluation_suite_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_kernels_task", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_kernels_task", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_figqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_lab_bench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_lab_bench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_latchbio_bioinformatics", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_latchbio_bioinformatics", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_latchbio_bioinformatics", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_latchbio_bioinformatics", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_legal_agent_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_legal_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_legal_agent_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_legal_agent_benchmark", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_linuxarena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_linuxarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_linuxarena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_linuxarena", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_llm_training", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_llm_training", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_1", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_2", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mask", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_mask", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_model_alignment_between_statements_and_knowledge", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_model_alignment_between_statements_and_knowledge", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_mathvista", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_prompt_injection_evaluation", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_mcp_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_medcalc_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_medcalc_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_mgsm", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_milu", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_milu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_mmmu_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_task_indic_language_understanding_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_multi_task_indic_language_understanding_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_multi_task_indic_language_understanding_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_multi_task_indic_language_understanding_benchmark", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_needle_in_a_haystack", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_novel_compiler", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_officeqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_officeqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_officeqa", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_openai_mrcr_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_openai_mrcr_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openrca", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_openrca", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_opinion_elicitation", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_opinion_elicitation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_oss_fuzz", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_oss_fuzz", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_oss_fuzz", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_oss_fuzz", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_osworld_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_osworld_verified", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_pretraining_documents", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_pretraining_documents", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_programbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_programbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_programbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_programbench", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_proteingym_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_proteingym_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_proteingym_hard", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_proteingym_hard", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_quadruped_reinforcement_learning", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_quadruped_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_random_chat_questions", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_random_chat_questions", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_real_world_finance", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_real_world_finance", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_real_world_finance_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_real_world_finance_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_real_world_finance_v2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_real_world_finance_v2", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_record_100q_hard", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_record_100q_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_record_2_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_record_2_bench_retail", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_research_sabotage", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_research_sabotage", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_research_sandbagging", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_research_sandbagging", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_reward_hack_prone_coding_tasks_v2", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_reward_hack_prone_coding_tasks_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_shade_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_shade_arena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_shade_arena", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_simpleqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_subversion_strategy", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_subversion_strategy", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_multilingual", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_multimodal", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_multimodal", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_multimodal", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_pro", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_verified", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_sycophancy_eval", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_sycophancy_eval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_synthetic_honeypots", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_synthetic_honeypots", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_terminal_bench_2_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_terminal_bench_2_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_terminal_bench_2_1", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_text_based_reinforcement_learning_task", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_text_based_reinforcement_learning_task", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_time_series_forecasting", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tool_use_prompt_injection_evaluation", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_tool_use_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_toolathlon", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_toolathlon", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_toolathlon", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_usamo_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_usamo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_usamo_2026", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_usamo_2026", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vals_ai_finance_agent", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_vals_ai_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_arena", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_vending_bench_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vmqa_vct", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_vmqa_vct", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webarena", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_webarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_webarena_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_webarena_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_wildchat", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_wildchat", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_xstest", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260630_introducing_claude_sonnet_5_benchmark_xstest", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260630_introducing_claude_sonnet_5", "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_abc_bench_fragment_design", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_abc_bench_fragment_design", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_fragment", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_fragment", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_abc_bench_liquid_handling", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_abc_bench_liquid_handling", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_liquid", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_liquid", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_abc_bench_screening_evasion", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_abc_bench_screening_evasion", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_screening", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_screening", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agentdojo", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_agentdojo", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_agentdojo", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_agentdojo", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_agentharm", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_agentharm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agentharm_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_agentharm_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_agentharm_verified", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_agentharm_verified", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agentic_cybermisuse", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_agentic_cybermisuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agentic_misalignment", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_agentic_misalignment", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_agentic_misalignment", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_agentic_misalignment", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_airs_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_airs_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_airs_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_airs_bench", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_alignment_faking", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_alignment_faking", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_alignment_faking", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_alignment_faking", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_arc_agi_2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_babyvision", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_babyvision", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_babyvision", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_babyvision", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biodesign_tools", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_biodesign_tools", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_biodesign_tools", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_biodesign_tools", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_biotier_refusal", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_biotier_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_biotier_refusal", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_biotier_refusal", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_catastrophic_cyber_misuse", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_catastrophic_cyber_misuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_catastrophic_cyber_misuse", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_catastrophic_cyber_misuse", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_charxiv_reasoning", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_charxiv_reasoning", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_charxiv_reasoning", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_chemical_agents_refusal", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_chemical_agents_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_chemical_agents_refusal", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_chemical_agents_refusal", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_curated_ctfs", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_curated_ctfs", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_curated_ctfs", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_curated_ctfs", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_cybench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cybench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cybench", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cyber_benign_use", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_cyber_benign_use", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cyber_misuse", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_cyber_misuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cyber_misuse_chat", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_cyber_misuse_chat", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cyber_misuse_chat", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cyber_misuse_chat", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyberbench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_cyberbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberbench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberbench", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_cybergym", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cybergym", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cybergym", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_cyberseceval_code_interpreter_abuse", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_cyberseceval_code_interpreter_abuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberseceval_code_interpreter_abuse", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberseceval_code_interpreter_abuse", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyberseceval_insecure_coding", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_cyberseceval_insecure_coding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberseceval_insecure_coding", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberseceval_insecure_coding", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cyscenariobench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_cyscenariobench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cyscenariobench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cyscenariobench", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deceptionbench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_deceptionbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_deceptionbench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_deceptionbench", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deepsearchqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_deepsearchqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_deepsearchqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_deepsearchqa", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deepswe_v1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_deepswe_v1_1", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_deepswe_v1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_deepswe_v1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_erqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_exploitgym", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_exploitgym", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_exploitgym", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_exploitgym", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_finance_agent_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_finance_agent_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_finance_agent_v2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_finance_agent_v2", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_fortress", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_fortress", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_fortress", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_fortress", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_frontierscience_research", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_frontierscience_research", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdm_situational_awareness", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_gdm_situational_awareness", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_gdm_situational_awareness", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_gdm_situational_awareness", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdm_stealth", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_gdm_stealth", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_gdm_stealth", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_gdm_stealth", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_gdpval_aa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_gdpval_aa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_gdpval_aa", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_gpqa_diamond", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_grayswan_art", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_grayswan_art", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_grayswan_art", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_grayswan_art", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_healthbench_hard", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_healthbench_hard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_healthbench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_healthbench_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_healthbench_pro", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_healthbench_pro", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_hle", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_humanitys_last_exam", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_humanitys_last_exam", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hle_calibration", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_hle_calibration", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_hle_calibration", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_hle_calibration", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hpct", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_hpct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_hpct", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_hpct", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_iheval", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_iheval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_impossiblebench", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_impossiblebench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_internal_sycophancy", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_internal_sycophancy", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_internal_sycophancy", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_internal_sycophancy", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_international_physics_olympiad_2025_theory", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_international_physics_olympiad_2025_theory", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_jobbench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_jobbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_jobbench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_jobbench", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_lab_bench_protocolqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_protocolqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_protocolqa", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_lab_bench_seqqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_seqqa", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_seqqa", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_livecodebench_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_livecodebench_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mask", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_mask", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_mask", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_mask", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mbct", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_mbct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_mbct", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_mbct", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_mcp_atlas", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_mcp_atlas", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_mcp_atlas", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_medxpertqa_multimodal", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_medxpertqa_multimodal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_medxpertqa_text", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_medxpertqa_text", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mle_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_mle_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_mmmu_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mrcr_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_mrcr_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_mrcr_v2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_mrcr_v2", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_or_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_or_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_or_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_or_bench", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_osworld_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_osworld_2_0", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_osworld_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_osworld_2", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_osworld_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_osworld_verified", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_osworld_verified", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_poly_guard_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_poly_guard_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_poly_guard_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_poly_guard_bench", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_prompted_evaluation_awareness", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_prompted_evaluation_awareness", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_redcode", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_redcode", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_redcode", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_redcode", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_save_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_save_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_save_bench", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_save_bench", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_screenspot_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_secure_code_generation", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_secure_code_generation", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_severe_cybermisuse_refusal", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_severe_cybermisuse_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_shade_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_shade_arena", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_shade_arena", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_shade_arena", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_simpleqa_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_simplevqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_social_engineering_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_social_engineering_evaluation", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_social_engineering_eval", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_social_engineering_eval", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_social_engineering_refusal", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_social_engineering_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_strongreject_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_strongreject_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_strongreject_v2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_strongreject_v2", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_swe_bench_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_bench_pro", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_swe_bench_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_swe_bench_verified_hard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_bench_verified_hard", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_bench_verified_hard", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_pi_agent", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_swe_pi_agent", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_pi_agent", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_pi_agent", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_pi_synthetic", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_swe_pi_synthetic", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_pi_synthetic", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_pi_synthetic", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tau_bench_telecom", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_tau_bench_telecom", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_terminal_bench_2_0", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_1", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_terminal_bench_2_1", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_terminal_bench_2_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_terminal_bench_2_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toolathlon_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_toolathlon_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_toolathlon_verified", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_toolathlon_verified", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vct", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_vct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_vct", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_vct", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_vibecodebench_v1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_vibecodebench_v1_1", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_vibecodebench_v1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_vibecodebench_v1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_webarena_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_webarena_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_webarena_verified", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_webarena_verified", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_widesearch", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_widesearch", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_widesearch", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_widesearch", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_wmdp_bio", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_wmdp_bio", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_wmdp_bio", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_wmdp_bio", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_wmdp_chem", "consecutive_reviewed_omissions": 0, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_wmdp_chem", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": "occurrence_meta_occurrence_meta_muse_spark_1_1_wmdp_chem", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_wmdp_chem", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_wmdp_cyber", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_wmdp_cyber", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_zerobench", "consecutive_reviewed_omissions": 1, "id": "status_release_meta_20260709_introducing_muse_spark_1_1_benchmark_zerobench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "model_id": "model_meta_model_meta_muse_spark_1_1", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_meta_20260709_introducing_muse_spark_1_1", "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agents_last_exam", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_agents_last_exam", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_agents_last_exam", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_agents_last_exam", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_arc_agi_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_arc_agi_2", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_artificial_analysis_coding_agent_index", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_artificial_analysis_coding_agent_index", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_artificial_analysis_coding_agent_index", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_artificial_analysis_coding_agent_index", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_artificial_analysis_intelligence_index", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_artificial_analysis_intelligence_index", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_artificial_analysis_intelligence_index", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_artificial_analysis_intelligence_index", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_automation_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_automationbench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_automationbench", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_benchcad", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_benchcad", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchcad", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchcad", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_big_finance_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_big_finance_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_big_finance_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_big_finance_bench", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_browsecomp", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_browsecomp", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_capture_the_flag_challenges", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_capture_the_flag_challenges", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_capture_the_flag_challenges", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_capture_the_flag_challenges", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deepswe", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_deepswe", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_deepswe", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_deepswe", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_exploitbench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_exploitbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_exploitgym", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_exploitgym", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontiermath_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_frontiermath_v2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_frontiermath_v2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_frontiermath_v2", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_gdpval_aa_v2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_gdpval_aa_v2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_gdpval_aa_v2", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_genebench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_genebench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_genebench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_genebench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_genebench_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_genebench_pro", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_gpqa_diamond", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_graphwalks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_graphwalks", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench_professional", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_healthbench_professional", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_healthbench_professional", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_healthbench_professional", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_kernelgen_1p", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_kernelgen_1p", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_kernelgen_1p", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_kernelgen_1p", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_medchembench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_medchembench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_medchembench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_medchembench", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_mmmu_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_mmmu_pro", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_management_consulting_tasks", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_openai_management_consulting_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_management_consulting_tasks", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_management_consulting_tasks", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_mrcr", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_mrcr", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_mrcr", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_openai_research_debugging_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_openai_research_debugging_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_research_debugging_evaluation", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_research_debugging_evaluation", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_osworld", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_osworld", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_osworld", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_sec_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_sec_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_sec_bench_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_sec_bench_pro", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_swe_bench_pro", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_terminal_bench", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_terminal_bench", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 0, "id": "status_release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchmark_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "occurrence_id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_toolathlon", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_toolathlon", "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_big_sleep_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_benchmark_big_sleep_evaluation", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_cyber_text_coding", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_cyber", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_big_sleep_evaluation", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_big_sleep_evaluation", "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chrome_production_commit_scanning_pipeline", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_benchmark_chrome_production_commit_scanning_pipeline", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_cyber_text_coding", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_cyber", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_chrome_commit_scan", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_chrome_commit_scan", "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_benchmark_cybergym", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_cyber_text_coding", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_cyber", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_cybergym", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_cybergym", "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_ai_text_leaderboard", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_arena_ai_text_leaderboard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_charxiv_reasoning", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_cybergym", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_cybergym", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_cybergym", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deepswe", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_deepswe", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_deepswe", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_deepswe", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdm_mrcr_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_gdm_mrcr_v2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_gdm_mrcr_v2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_gdm_mrcr_v2", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_gdpval_aa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval_aa_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_gdpval_aa_v2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_gdpval_aa_v2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_gdpval_aa_v2", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 17, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_matharena_apex", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_matharena_apex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_mcp_atlas", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mle_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_mle_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_mle_bench", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_mle_bench", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 12, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_osworld_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_osworld_verified", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_osworld_verified", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_simpleqa_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_swe_bench_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_swe_bench_pro", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_terminal_bench_2_0", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_terminal_bench_2", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_terminal_bench_2", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2025", "consecutive_reviewed_omissions": 8, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_vending_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_video_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aa_briefcase", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_aa_briefcase", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_aa_briefcase", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_aa_briefcase", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aa_coding_agent_index", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_aa_coding_agent_index", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_aa_coding_agent_index", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_aa_coding_agent_index", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_aav_capsid_packaging_prediction", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_aav_capsid_packaging_prediction", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_aider_polyglot", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_aider_polyglot", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_aime_2025", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_1", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_2", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_arc_agi_3", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_3", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_3", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_3", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_arxivmath", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_arxivmath", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arxivmath", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arxivmath", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_automation_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_automationbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_automationbench", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bbq", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_bbq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_bias_benchmark_for_question_answering", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_bias_benchmark_for_question_answering", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bench", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_benchcad", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_benchcad", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_benchcad", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_benchcad", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_biglaw_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_biglaw_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_biomysterybench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_biomysterybench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_biomysterybench", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_biomysterybench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_biomysterybench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_biopipelinebench", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_biopipelinebench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_biopipelinebench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_biopipelinebench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_black_box_rna_sequence_modeling_and_design", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_black_box_rna_sequence_modeling_and_design", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_browsecomp", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_browsecomp", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_browsecomp_plus", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_browsecomp_plus", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartmuseum", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_chartmuseum", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartography", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_chartography", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_chartography", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_chartography", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartqapro", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_chartqapro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cloningscenarios", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_creative_biology", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cursorbench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_cursorbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cursorbench_3_2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_cursorbench_3_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_cursorbench_3_2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_cursorbench_3_2", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cybench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cyscenariobench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_cyscenariobench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_cyscenariobench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_cyscenariobench", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deep_research_accuracy_completeness_and_objectivity", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_deep_research_accuracy_completeness_and_objectivity", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deep_research_accuracy_completeness_and_objectivity", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deep_research_accuracy_completeness_and_objectivity", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deepsearchqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deepsearchqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deepsearchqa", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deepswe_v1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_deepswe_v1_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deepswe_v1_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deepswe_v1_1", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_draco", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_draco", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_exploitbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_exploitbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_exploitbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_exploitbench", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_exploitgym", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_exploitgym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_exploitgym", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_exploitgym", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_figqa", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_finance_agent", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_firefox_147_exploit_evaluation", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_firefox_147_exploit_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_firefox_147_exploit_evaluation", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_firefox_147_exploit_evaluation", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_firefox_147_exploitation_evaluation", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_firefox_147_exploitation_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_firefox_exploit_evaluation", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_firefox_exploit_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontier_bench_v0_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_frontier_bench_v0_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_frontier_bench_v0_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_frontier_bench_v0_1", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontiercode_v1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_frontiercode_v1_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_frontiercode_v1_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_frontiercode_v1_1", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontierswe", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_frontierswe", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdp_pdf", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_gdp_pdf", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_gdp_pdf", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_gdp_pdf", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval_aa_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_gdpval_aa_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_gdpval_aa_v2", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_gdpval_aa_v2", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_global_mmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_global_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_global_mmlu", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_global_mmlu", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_graphwalks_bfs_256k", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_graphwalks_bfs_256k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_graphwalks_parents_256k", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_graphwalks_parents_256k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_healthbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_healthbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_healthbench", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench_professional", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_healthbench_professional", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_healthbench_professional", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_healthbench_professional", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_hle", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_humanity_s_last_exam", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_imo_2026", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_imo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_imo_2026", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_imo_2026", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_include", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_include", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_include", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_kernel_optimization", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_kernel_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_figqa", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_lab_bench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lab_bench_seqqa", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_lab_bench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_labbench2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_labbench2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_latchbio_bioinformatics", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_latchbio_bioinformatics", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_latchbio_bioinformatics", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_latchbio_bioinformatics", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_legal_agent_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_legal_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_legal_agent_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_legal_agent_benchmark", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_linuxarena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_linuxarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_linuxarena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_linuxarena", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_llm_training_optimization", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_llm_training_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_1", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_long_form_virology_task_2", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mask", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_mask", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_model_alignment_between_statements_and_knowledge", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_model_alignment_between_statements_and_knowledge", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_mcp_atlas", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_mcp_atlas", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_minimal_linuxbench", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_minimal_linuxbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_mmmu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_mmmu_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multi_task_indic_language_understanding_benchmark", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_multi_task_indic_language_understanding_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_multi_task_indic_language_understanding_benchmark", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_multi_task_indic_language_understanding_benchmark", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_novel_compiler", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_officeqa", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_officeqa", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_officeqa", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_officeqa_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_officeqa_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_online_mind2web", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_online_mind2web", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_mrcr_v2", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_openai_mrcr_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openrca", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_openrca", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_oss_fuzz", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_oss_fuzz", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_oss_fuzz", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_oss_fuzz", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_osworld_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_osworld_2_0", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_osworld_2_0", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_programbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_programbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_programbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_programbench", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_proteingym_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_proteingym_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_proteingym_hard", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_proteingym_hard", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_quadruped_locomotion", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_quadruped_locomotion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_rakuten_swe_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_rakuten_swe_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_real_world_finance", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_real_world_finance", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_airline_corrected", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_record_2_bench_airline_corrected", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_airline_original", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_record_2_bench_airline_original", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_retail", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_record_2_bench_retail", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_2_bench_telecom", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_record_2_bench_telecom", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_riemannbench", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_riemannbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_riemannbench", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_riemannbench", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_screenspot_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_securebio_dna_synthesis_screening_evasion", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_securebio_dna_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_securebio_virology_capability_test", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_securebio_virology_capability_test", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_shade_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_shade_arena", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_shade_arena", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_spreadsheetbench", "consecutive_reviewed_omissions": 4, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_spreadsheetbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_super_agent_benchmark", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_super_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_multilingual", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_multimodal", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_multimodal", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_multimodal", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_pro", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_verified", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_synthesis_screening_evasion", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 2, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_1", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_terminal_bench_2_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_text_based_reinforcement_learning", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_text_based_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_time_series_forecasting", "consecutive_reviewed_omissions": 5, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_toolathlon", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_toolathlon_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_toolathlon_verified", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_toolathlon_verified", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_usamo_2026", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_usamo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 1, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_webarena", "consecutive_reviewed_omissions": 3, "id": "status_release_anthropic_20260724_introducing_claude_opus_5_benchmark_webarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_anthropic_20260724_introducing_claude_opus_5", "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_agents_last_exam", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_agents_last_exam", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_artificial_analysis_coding_agent_index", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_artificial_analysis_coding_agent_index", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_artificial_analysis_intelligence_index", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_artificial_analysis_intelligence_index", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_automation_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_benchcad", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_benchcad", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_big_finance_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_big_finance_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_capture_the_flag_challenges", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_capture_the_flag_challenges", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_deepswe", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_deepswe", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_exploitbench", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_exploitbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_exploitgym", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_exploitgym", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontiermath_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_frontiermath_v2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval_aa_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_gdpval_aa_v2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_genebench", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_genebench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_genebench_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_genebench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_healthbench_professional", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_healthbench_professional", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_kernelgen_1p", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_kernelgen_1p", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_medchembench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_medchembench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_management_consulting_tasks", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_openai_management_consulting_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_mrcr", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_research_debugging_evaluation", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_openai_research_debugging_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_osworld", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_sec_bench_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_sec_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 1, "id": "status_release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5_benchmark_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "release_id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agents_last_exam", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_agents_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_agents_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_agents_last_exam", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_apex", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_apex", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_apex_shortlist", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_apex_shortlist", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_automation_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_automationbench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_automationbench", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_bigcodebench", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_bigcodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_chinese_simpleqa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_chinese_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_corpusqa_1m", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_corpusqa_1m", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_cybergym", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_cybergym", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_cybergym", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deepswe", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_deepswe", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_deepswe", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_deepswe", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_dsbench_fullstack", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_dsbench_fullstack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_dsbench_fullstack", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_dsbench_fullstack", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_dsbench_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_dsbench_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_dsbench_hard", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_dsbench_hard", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_facts_parametric", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_facts_parametric", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_gdpval_aa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_hle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_hmmt_2026_feb", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_hmmt_2026_feb", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_imoanswerbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcpatlas", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_mcpatlas", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcpatlas_public", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_mcpatlas_public", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_mmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mrcr_1m", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_mrcr_1m", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_multiloko", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_multiloko", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_nl2repo", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_nl2repo", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_nl2repo", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_simpleqa_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_supergpqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_swe_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_terminal_bench_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_1", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_terminal_bench_2_1", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_terminal_bench_2_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_terminal_bench_2_1", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_toolathlon", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_toolathlon_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_toolathlon_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_toolathlon_verified", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 1, "id": "status_release_deepseek_20260731_deepseek_v4_flash_update_benchmark_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260731_deepseek_v4_flash_update", "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_agents_last_exam", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_agents_last_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_agents_last_exam", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_agents_last_exam", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_androidbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_androidbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_androidbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_androidbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_androidworld", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_androidworld", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_automation_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_automation_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_automation_bench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_babyvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_babyvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_babyvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_babyvision", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_cc_ocr_bench_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_cc_ocr_bench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_cc_ocr_bench_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_cc_ocr_bench_v2", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_charxiv_rq", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_charxiv_rq", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_claweval_mm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_claweval_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_claweval_mm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_claweval_mm", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_coco", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_coco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_coco", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_coco", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_countqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_countqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_countqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_countqa", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_coworkbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_coworkbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_coworkbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_coworkbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deepswe_1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_deepswe_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_deepswe_1_1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_deepswe_1_1", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_dense200", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_dense200", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_dense200", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_dense200", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_egolife_w_mem", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_egolife_w_mem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_egolife_w_mem", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_egolife_w_mem", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_erqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_erqa", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_frontierswe", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_frontierswe", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_frontierswe", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_frontierswe", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_gpqa_diamond", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_healthbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_healthbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_healthbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_healthbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hipho", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_hipho", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hipho", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hipho", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle_vl_w_tools", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_hle_vl_w_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle_vl_w_tools", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle_vl_w_tools", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle_with_tools", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle_with_tools", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle_with_tools", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_ifbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_ifbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_jobbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_jobbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_jobbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_jobbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lingoqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_lingoqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lingoqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lingoqa", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_logicvista", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_logicvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_logicvista", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_logicvista", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_longbench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_longbench_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_longbench_v2", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lvbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lvbench_w_mem", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_lvbench_w_mem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lvbench_w_mem", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lvbench_w_mem", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_madqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_madqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_madqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_madqa", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mathvision", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_medxpertqa_mm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_medxpertqa_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_medxpertqa_mm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_medxpertqa_mm", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mls_bench_lite", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_mls_bench_lite", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mls_bench_lite", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mls_bench_lite", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mlvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mlvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mlvu", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmmu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmmu_pro", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmstar", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmstar", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmstar", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmvu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_mmvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmvu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmvu", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mobileworld", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_mobileworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mobileworld", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mobileworld", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mrcr_v2", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_mrcr_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mrcr_v2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mrcr_v2", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mtvqa_test", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_mtvqa_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mtvqa_test", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mtvqa_test", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_nl2repo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_nl2repo", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ocr_bench_v2_en_zh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_ocr_bench_v2_en_zh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_ocr_bench_v2_en_zh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_ocr_bench_v2_en_zh", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_omnidocbench_1_5", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_omnidocbench_1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_omnidocbench_1_5", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_onemillion_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_onemillion_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_onemillion_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_onemillion_bench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_osworld_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_osworld_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_osworld_2_0", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_osworld_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_osworld_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_osworld_verified", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_paperbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_paperbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_paperbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_paperbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_parametric_cad_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_parametric_cad_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_parametric_cad_bench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_parametric_cad_bench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_perceptionbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_perceptionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_perceptionbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_perceptionbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_phyx", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_phyx", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_phyx", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_phyx", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_plawbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_plawbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_plawbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_plawbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_pmc_vqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_pmc_vqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_pmc_vqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_pmc_vqa", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_prbench_finance", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_prbench_finance", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_prbench_finance", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_prbench_finance", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_prbench_legal", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_prbench_legal", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_prbench_legal", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_prbench_legal", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_presentbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_presentbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_presentbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_presentbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenblenderbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_qwenblenderbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenblenderbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenblenderbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenqoderbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_qwenqoderbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenqoderbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenqoderbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenreactbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_qwenreactbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenreactbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenreactbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwensvgbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_qwensvgbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwensvgbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwensvgbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenswebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_qwenswebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenswebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenswebench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_qwenvisualoffice", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_qwenvisualoffice", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenvisualoffice", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenvisualoffice", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_realworldqa", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_recreationbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_recreationbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_recreationbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_recreationbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_refadv_s", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_refadv_s", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_refadv_s", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_refadv_s", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_screenspot_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_screenspot_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_screenspot_pro", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_simplevqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_simplevqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_simplevqa", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_skillsbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_skillsbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_skillsbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_slake", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_slake", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_slake", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_slake", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_surds", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_surds", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_surds", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_surds", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_swe_bench_pro", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench_2_1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_terminal_bench_2_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_terminal_bench_2_1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_terminal_bench_2_1", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toolathlon_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_toolathlon_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_toolathlon_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_toolathlon_verified", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_tvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_tvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_tvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_tvbench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videodr_w_search", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_videodr_w_search", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videodr_w_search", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videodr_w_search", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_v2_w_sub", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_videomme_v2_w_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videomme_v2_w_sub", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videomme_v2_w_sub", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videomme_with_subtitles", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videomme_with_subtitles", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videomme_with_subtitles", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_videommmu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videommmu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videommmu", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_visfactor", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_visfactor", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_visfactor", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_visfactor", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vision2web", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_vision2web", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_vision2web", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_vision2web", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vlmsarebiased", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_vlmsarebiased", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_vlmsarebiased", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_vlmsarebiased", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_webarena_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_webarena_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_webarena_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_webarena_verified", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_widesearch", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_widesearch", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_widesearch", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_workspacebench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_workspacebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_workspacebench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_workspacebench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_worldvqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_worldvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_worldvqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_worldvqa", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_zerobench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_zerobench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_zerobench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_zerobench", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_zerobench_sub", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260803_qwen3_8_benchmark_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_zerobench_sub", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_zerobench_sub", "release_qwen_20260803_qwen3_8" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agents_last_exam", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_agents_last_exam", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_artificial_analysis_coding_agent_index", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_artificial_analysis_coding_agent_index", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_artificial_analysis_intelligence_index", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_artificial_analysis_intelligence_index", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_automation_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_benchcad", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_benchcad", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_big_finance_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_big_finance_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_capture_the_flag_challenges", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_capture_the_flag_challenges", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_deepswe", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_deepswe", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_exploitbench", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_exploitbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_exploitgym", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_exploitgym", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontiermath_v2", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_frontiermath_v2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval_aa_v2", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_gdpval_aa_v2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_genebench", "consecutive_reviewed_omissions": 3, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_genebench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_genebench_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_genebench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_graphwalks", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_healthbench_professional", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_healthbench_professional", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_kernelgen_1p", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_kernelgen_1p", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_medchembench", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_medchembench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_management_consulting_tasks", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_openai_management_consulting_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_mrcr", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openai_research_debugging_evaluation", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_openai_research_debugging_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_osworld", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_sec_bench_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_sec_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 2, "id": "status_release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding_benchmark_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "release_id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_agents_last_exam", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_agents_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_agents_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_agents_last_exam", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_agieval", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_apex", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_apex", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_apex_shortlist", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_apex_shortlist", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_automation_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_automationbench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_automationbench", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_bigcodebench", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_bigcodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_browsecomp", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_c_eval", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chinese_simpleqa", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_chinese_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cluewsc", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmath", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cmmlu", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_codeforces", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_corpusqa_1m", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_corpusqa_1m", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_cybergym", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_cybergym", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_cybergym", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_deepswe", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_deepswe", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_deepswe", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_deepswe", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dsbench_fullstack", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_dsbench_fullstack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_dsbench_fullstack", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_dsbench_fullstack", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_dsbench_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_dsbench_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_dsbench_hard", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_dsbench_hard", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_facts_parametric", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_facts_parametric", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_gdpval_aa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_hle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_humanity_s_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_humanity_s_last_exam", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_hmmt_2026_feb", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_hmmt_2026_feb", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_imoanswerbench", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_imoanswerbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_longbench_v2", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_massive_multitask_language_understanding", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcpatlas", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_mcpatlas", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcpatlas_public", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_mcpatlas_public", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_mmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mrcr_1m", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_mrcr_1m", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_multiloko", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_multiloko", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_nl2repo", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_nl2repo", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_nl2repo", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_simpleqa_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_supergpqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_pro", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_swe_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_terminal_bench_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_1", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_terminal_bench_2_1", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_terminal_bench_2_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_terminal_bench_2_1", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_toolathlon", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_toolathlon", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_toolathlon_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_toolathlon_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_toolathlon_verified", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_toolathlon_verified", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "continued" ] }, { "benchmark_id": "benchmark_triviaqa", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_winogrande", "consecutive_reviewed_omissions": 2, "id": "status_release_deepseek_20260813_deepseek_v4_pro_update_benchmark_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_deepseek_20260813_deepseek_v4_pro_update", "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_activitynet_qa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_ai2d", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_aime_2025", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_amc_2022_23", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_amc_2022_23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arc_agi_2", "consecutive_reviewed_omissions": 4, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_arena_ai_text_leaderboard", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_arena_ai_text_leaderboard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_automatic_speech_recognition_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_automation_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_automationbench", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_automationbench", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_bbh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_betterchartqa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_betterchartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_blink", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chartqa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_charxiv_reasoning", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_charxiv_reasoning", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_chemicaldiagramqa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_chemicaldiagramqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_cybergym", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_cybergym", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_deepswe", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_deepswe", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_deepswe_v1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_deepswe_v1_1", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_deepswe_v1_1", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_deepswe_v1_1", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_docvqa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_drop", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_dude", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_egoschema", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_frontiercode_v2_main", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_frontiercode_v2_main", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_frontiercode_v2_main", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_frontiercode_v2_main", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_functional_math", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdm_mrcr_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_gdm_mrcr_v2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdp_pdf", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_gdp_pdf", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_gdp_pdf", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_gdp_pdf", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gdpval_aa", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_gdpval_aa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gdpval_aa_v2", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_gdpval_aa_v2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hellaswag", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hiddenmath", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_hle", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_humaneval", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_humaneval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_infographicvqa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_livecodebench", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_lmarena", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_machine_translation_from_one_book", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_massive_multitask_language_understanding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 18, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_matharena_apex", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_matharena_apex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mathvista", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mcp_atlas", "consecutive_reviewed_omissions": 2, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_mcp_atlas", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mle_bench", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_mle_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 13, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_mmmu_pro", "consecutive_reviewed_omissions": 3, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_natural2code", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_needle_in_a_haystack", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_needle_in_a_haystack", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_openeqa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_osworld_verified", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_osworld_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_physicsfinals", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_physicsfinals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_record_1h_videoqa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_record_1h_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_simpleqa_verified", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_simpleqa_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_swe_bench_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_swe_bench_verified", "consecutive_reviewed_omissions": 5, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_tat_dqa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_terminal_bench_2_0", "consecutive_reviewed_omissions": 1, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_terminal_bench_2_0", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "dormant_in_public_reporting" ] }, { "benchmark_id": "benchmark_textvqa", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_usamo_2025", "consecutive_reviewed_omissions": 9, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_v_bench", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_v_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vatex_zh", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vending_bench_2", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_vending_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_video_mmmu", "consecutive_reviewed_omissions": 6, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_video_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_videomme", "consecutive_reviewed_omissions": 10, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_videomme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_vqav2", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_webdev_arena", "consecutive_reviewed_omissions": 0, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_webdev_arena", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_webdev_arena", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "returning" ] }, { "benchmark_id": "benchmark_wmt23", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_youcook2", "consecutive_reviewed_omissions": 19, "id": "status_release_google_deepmind_20260813_introducing_gemini_3_7_flash_benchmark_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "occurrence_id": null, "provenance": { "record_ids": [ "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "status_ids": [ "not_reported_in_reviewed_successor", "outdated_in_public_reporting" ] }, { "benchmark_id": "benchmark_agents_last_exam", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260821_deepseek_v4_flash_vision_exp_release_benchmark_agents_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_agents_last_exam", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_agents_last_exam", "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_apexbench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260821_deepseek_v4_flash_vision_exp_release_benchmark_apexbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_apexbench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_apexbench", "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_automation_bench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260821_deepseek_v4_flash_vision_exp_release_benchmark_automation_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_automationbench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_automationbench", "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_chartography", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260821_deepseek_v4_flash_vision_exp_release_benchmark_chartography", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_chartography", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_chartography", "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deepswe", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260821_deepseek_v4_flash_vision_exp_release_benchmark_deepswe", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_deepswe", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_deepswe", "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_dsbench_hard", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260821_deepseek_v4_flash_vision_exp_release_benchmark_dsbench_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_dsbench_hard", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_dsbench_hard", "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260821_deepseek_v4_flash_vision_exp_release_benchmark_nl2repo", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_nl2repo", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_nl2repo", "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_terminal_bench_2_1", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260821_deepseek_v4_flash_vision_exp_release_benchmark_terminal_bench_2_1", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_terminal_bench_2_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_terminal_bench_2_1", "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_zerobench", "consecutive_reviewed_omissions": 0, "id": "status_release_deepseek_20260821_deepseek_v4_flash_vision_exp_release_benchmark_zerobench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "occurrence_id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_zerobench", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_zerobench", "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_agents_last_exam", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_agents_last_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_agents_last_exam", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_agents_last_exam", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_androidworld", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_androidworld", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_androidworld", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_bbh", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_bbh", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_bbh", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_charxiv_rq", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_charxiv_rq", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_charxiv_rq", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_claweval_mm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_claweval_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_claweval_mm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_claweval_mm", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_coworkbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_coworkbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_coworkbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_coworkbench", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_deepswe_1_1", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_deepswe_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_deepswe_1_1", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_deepswe_1_1", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_erqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_erqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_erqa", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_evalplus", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_evalplus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_evalplus", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gpqa", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gpqa_diamond", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gpqa_diamond", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gpqa_diamond", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_gsm8k", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gsm8k", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gsm8k", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_hle", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_hle", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_hle", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_ifbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_ifbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_ifbench", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_include", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_include", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_include", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_jobbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_jobbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_jobbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_jobbench", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_livecodebench_v6", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_livecodebench_v6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_livecodebench_v6", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_lvbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_lvbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_lvbench", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_massive_multitask_language_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_math", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_math", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mathvision", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mathvision", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mathvision", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mgsm", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mgsm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mgsm", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu_pro", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmlu_redux", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu_redux", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu_redux", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_mmmlu", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmmlu", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmmlu", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_multipl_e", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_multipl_e", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_multipl_e", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_nl2repo", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_nl2repo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_nl2repo", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_osworld_2_0", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_osworld_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_osworld_2_0", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_osworld_2_0", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_realworldqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_realworldqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_realworldqa", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_recreationbench", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_recreationbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_recreationbench", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_recreationbench", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_supergpqa", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_supergpqa", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_supergpqa", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_multilingual", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swe_bench_multilingual", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swe_bench_multilingual", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swe_bench_pro", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swe_bench_pro", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swe_bench_pro", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_swebench_pretrain", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_swebench_pretrain", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swebench_pretrain", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swebench_pretrain", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_toolathlon_verified", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_toolathlon_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_toolathlon_verified", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_toolathlon_verified", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] }, { "benchmark_id": "benchmark_vision2web", "consecutive_reviewed_omissions": 0, "id": "status_release_qwen_20260826_qwen3_8_flash_next_benchmark_vision2web", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "occurrence_id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_vision2web", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_vision2web", "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "derived_status", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "status_ids": [ "first_reported" ] } ], "generated_from": { "catalog": "data/catalog.yaml", "compiler": "scripts/compile.py", "definitions": "data/definitions.yaml" }, "labs": [ { "id": "anthropic", "name": "Anthropic", "official_domains": [ "anthropic.com", "assets.anthropic.com", "www-cdn.anthropic.com", "www.anthropic.com" ], "provenance": { "record_ids": [ "anthropic" ], "record_type": "labs", "source_file": "data/catalog.yaml" } }, { "id": "deepseek", "name": "DeepSeek", "official_domains": [ "api-docs.deepseek.com", "github.com", "huggingface.co" ], "provenance": { "record_ids": [ "deepseek" ], "record_type": "labs", "source_file": "data/catalog.yaml" } }, { "id": "google_deepmind", "name": "Google DeepMind", "official_domains": [ "ai.google.dev", "blog.google", "deepmind.google", "developers.googleblog.com", "research.google", "services.google.com", "storage.googleapis.com" ], "provenance": { "record_ids": [ "google_deepmind" ], "record_type": "labs", "source_file": "data/catalog.yaml" } }, { "id": "meta", "name": "Meta", "official_domains": [ "ai.meta.com", "arxiv.org", "github.com", "llama.com", "www.llama.com" ], "provenance": { "record_ids": [ "meta" ], "record_type": "labs", "source_file": "data/catalog.yaml" } }, { "id": "openai", "name": "OpenAI", "official_domains": [ "deploymentsafety.openai.com", "openai.com" ], "provenance": { "record_ids": [ "openai" ], "record_type": "labs", "source_file": "data/catalog.yaml" } }, { "id": "qwen", "name": "Qwen", "official_domains": [ "qwen.ai" ], "provenance": { "record_ids": [ "qwen" ], "record_type": "labs", "source_file": "data/catalog.yaml" } } ], "lineages": [ { "id": "lineage_anthropic_claude_3_5_family", "lab_id": "anthropic", "modality_class": "vision_language", "name": "Claude 3.5 family", "provenance": { "record_ids": [ "lineage_anthropic_claude_3_5_family" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_anthropic_claude_3_family", "lab_id": "anthropic", "modality_class": "vision_language", "name": "Claude 3 family", "provenance": { "record_ids": [ "lineage_anthropic_claude_3_family" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_anthropic_claude_4_family", "lab_id": "anthropic", "modality_class": "vision_language", "name": "Claude 4 family", "provenance": { "record_ids": [ "lineage_anthropic_claude_4_family" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_anthropic_claude_fable_mythos", "lab_id": "anthropic", "modality_class": "vision_language", "name": "Claude Fable and Mythos", "provenance": { "record_ids": [ "lineage_anthropic_claude_fable_mythos" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_anthropic_claude_haiku", "lab_id": "anthropic", "modality_class": "vision_language", "name": "Claude Haiku", "provenance": { "record_ids": [ "lineage_anthropic_claude_haiku" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_anthropic_claude_opus", "lab_id": "anthropic", "modality_class": "vision_language", "name": "Claude Opus", "provenance": { "record_ids": [ "lineage_anthropic_claude_opus" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_anthropic_claude_sonnet", "lab_id": "anthropic", "modality_class": "vision_language", "name": "Claude Sonnet", "provenance": { "record_ids": [ "lineage_anthropic_claude_sonnet" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_anthropic_claude_sonnet_reasoning", "lab_id": "anthropic", "modality_class": "vision_language", "name": "Claude Sonnet reasoning", "provenance": { "record_ids": [ "lineage_anthropic_claude_sonnet_reasoning" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_deepseek_deepseek_coder_v2", "lab_id": "deepseek", "modality_class": "text", "name": "Deepseek Coder V2", "provenance": { "record_ids": [ "lineage_deepseek_deepseek_coder_v2" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_deepseek_deepseek_r1", "lab_id": "deepseek", "modality_class": "text", "name": "Deepseek R1", "provenance": { "record_ids": [ "lineage_deepseek_deepseek_r1" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_deepseek_deepseek_v2", "lab_id": "deepseek", "modality_class": "text", "name": "Deepseek V2", "provenance": { "record_ids": [ "lineage_deepseek_deepseek_v2" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_deepseek_deepseek_v2_5", "lab_id": "deepseek", "modality_class": "text", "name": "Deepseek V2 5", "provenance": { "record_ids": [ "lineage_deepseek_deepseek_v2_5" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_deepseek_deepseek_v3", "lab_id": "deepseek", "modality_class": "text", "name": "Deepseek V3", "provenance": { "record_ids": [ "lineage_deepseek_deepseek_v3" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_deepseek_deepseek_v3_speciale", "lab_id": "deepseek", "modality_class": "text", "name": "Deepseek V3 Speciale", "provenance": { "record_ids": [ "lineage_deepseek_deepseek_v3_speciale" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_deepseek_deepseek_v4", "lab_id": "deepseek", "modality_class": "text", "name": "Deepseek V4", "provenance": { "record_ids": [ "lineage_deepseek_deepseek_v4" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_deepseek_deepseek_v4_vision", "lab_id": "deepseek", "modality_class": "vision_language", "name": "Deepseek V4 Vision", "provenance": { "record_ids": [ "lineage_deepseek_deepseek_v4_vision" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_google_deepmind_google_deepmind_diffusiongemma_text_general", "lab_id": "google_deepmind", "modality_class": "text", "name": "DiffusionGemma text models", "provenance": { "record_ids": [ "lineage_google_deepmind_google_deepmind_diffusiongemma_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "lab_id": "google_deepmind", "modality_class": "vision_language", "name": "Gemini computer-use models", "provenance": { "record_ids": [ "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_google_deepmind_google_deepmind_gemini_cyber_text_coding", "lab_id": "google_deepmind", "modality_class": "text", "name": "Gemini cyber-defense models", "provenance": { "record_ids": [ "lineage_google_deepmind_google_deepmind_gemini_cyber_text_coding" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "lab_id": "google_deepmind", "modality_class": "vision_language", "name": "Gemini general frontier models", "provenance": { "record_ids": [ "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "lab_id": "google_deepmind", "modality_class": "vision_language", "name": "Gemini Deep Think reasoning models", "provenance": { "record_ids": [ "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "lab_id": "google_deepmind", "modality_class": "text", "name": "Gemma open models", "provenance": { "record_ids": [ "lineage_google_deepmind_google_deepmind_gemma_general_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "lab_id": "google_deepmind", "modality_class": "vision_language", "name": "Gemma open models", "provenance": { "record_ids": [ "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_google_deepmind_google_deepmind_medgemma_vision_language_general", "lab_id": "google_deepmind", "modality_class": "vision_language", "name": "MedGemma healthcare models", "provenance": { "record_ids": [ "lineage_google_deepmind_google_deepmind_medgemma_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "lab_id": "google_deepmind", "modality_class": "text", "name": "T5Gemma encoder-decoder models", "provenance": { "record_ids": [ "lineage_google_deepmind_google_deepmind_t5gemma_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "lab_id": "google_deepmind", "modality_class": "text", "name": "VaultGemma private models", "provenance": { "record_ids": [ "lineage_google_deepmind_google_deepmind_vaultgemma_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_meta_lineage_meta_chameleon", "lab_id": "meta", "modality_class": "vision_language", "name": "Chameleon", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_chameleon" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_meta_lineage_meta_dynamic_blt", "lab_id": "meta", "modality_class": "text", "name": "Dynamic Blt", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_dynamic_blt" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_meta_lineage_meta_large_concept_models", "lab_id": "meta", "modality_class": "text", "name": "Large Concept Models", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_large_concept_models" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_meta_lineage_meta_layerskip_code", "lab_id": "meta", "modality_class": "text", "name": "Layerskip Code", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_layerskip_code" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_meta_lineage_meta_layerskip_llama", "lab_id": "meta", "modality_class": "text", "name": "Layerskip Llama", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_layerskip_llama" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_meta_lineage_meta_llama_instruct", "lab_id": "meta", "modality_class": "text", "name": "Llama Instruct", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_llama_instruct" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "instruct" }, { "id": "lineage_meta_lineage_meta_llama_text", "lab_id": "meta", "modality_class": "text", "name": "Llama Text", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_llama_text" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_meta_lineage_meta_llama_vision", "lab_id": "meta", "modality_class": "vision_language", "name": "Llama Vision", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_llama_vision" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_meta_lineage_meta_multi_token_prediction", "lab_id": "meta", "modality_class": "text", "name": "Multi Token Prediction", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_multi_token_prediction" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_meta_lineage_meta_muse", "lab_id": "meta", "modality_class": "vision_language", "name": "Muse", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_muse" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_meta_lineage_meta_plm", "lab_id": "meta", "modality_class": "vision_language", "name": "Plm", "provenance": { "record_ids": [ "lineage_meta_lineage_meta_plm" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_openai_computer_using_agent", "lab_id": "openai", "modality_class": "vision_language", "name": "Computer Using Agent", "provenance": { "record_ids": [ "lineage_openai_computer_using_agent" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_openai_gpt_4_1_general", "lab_id": "openai", "modality_class": "vision_language", "name": "Gpt 4 1 General", "provenance": { "record_ids": [ "lineage_openai_gpt_4_1_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_openai_gpt_4_general_text", "lab_id": "openai", "modality_class": "text", "name": "Gpt 4 General Text", "provenance": { "record_ids": [ "lineage_openai_gpt_4_general_text" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_openai_gpt_4o_general", "lab_id": "openai", "modality_class": "vision_language", "name": "Gpt 4O General", "provenance": { "record_ids": [ "lineage_openai_gpt_4o_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_openai_gpt_5_6_general", "lab_id": "openai", "modality_class": "vision_language", "name": "Gpt 5 6 General", "provenance": { "record_ids": [ "lineage_openai_gpt_5_6_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_openai_gpt_5_general", "lab_id": "openai", "modality_class": "vision_language", "name": "Gpt 5 General", "provenance": { "record_ids": [ "lineage_openai_gpt_5_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_openai_gpt_oss_reasoning", "lab_id": "openai", "modality_class": "text", "name": "Gpt Oss Reasoning", "provenance": { "record_ids": [ "lineage_openai_gpt_oss_reasoning" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_openai_gpt_rosalind", "lab_id": "openai", "modality_class": "vision_language", "name": "Gpt Rosalind", "provenance": { "record_ids": [ "lineage_openai_gpt_rosalind" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_openai_openai_codex_text", "lab_id": "openai", "modality_class": "text", "name": "Openai Codex Text", "provenance": { "record_ids": [ "lineage_openai_openai_codex_text" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_openai_openai_codex_vision", "lab_id": "openai", "modality_class": "vision_language", "name": "Openai Codex Vision", "provenance": { "record_ids": [ "lineage_openai_openai_codex_vision" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_openai_openai_daybreak", "lab_id": "openai", "modality_class": "vision_language", "name": "Openai Daybreak", "provenance": { "record_ids": [ "lineage_openai_openai_daybreak" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_openai_openai_o1_reasoning", "lab_id": "openai", "modality_class": "text", "name": "Openai O1 Reasoning", "provenance": { "record_ids": [ "lineage_openai_openai_o1_reasoning" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_openai_openai_o1_reasoning_vision", "lab_id": "openai", "modality_class": "vision_language", "name": "Openai O1 Reasoning Vision", "provenance": { "record_ids": [ "lineage_openai_openai_o1_reasoning_vision" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_openai_openai_o3_o4_reasoning_vision", "lab_id": "openai", "modality_class": "vision_language", "name": "Openai O3 O4 Reasoning Vision", "provenance": { "record_ids": [ "lineage_openai_openai_o3_o4_reasoning_vision" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_openai_openai_o3_reasoning_text", "lab_id": "openai", "modality_class": "text", "name": "Openai O3 Reasoning Text", "provenance": { "record_ids": [ "lineage_openai_openai_o3_reasoning_text" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_qwen_lineage_qwen1_5_text_coding", "lab_id": "qwen", "modality_class": "text", "name": "Qwen1 5 Text Coding", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen1_5_text_coding" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_qwen_lineage_qwen1_5_text_general", "lab_id": "qwen", "modality_class": "text", "name": "Qwen1 5 Text General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen1_5_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen2_5_text_coding", "lab_id": "qwen", "modality_class": "text", "name": "Qwen2 5 Text Coding", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen2_5_text_coding" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_qwen_lineage_qwen2_5_text_general", "lab_id": "qwen", "modality_class": "text", "name": "Qwen2 5 Text General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen2_5_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "lab_id": "qwen", "modality_class": "text", "name": "Qwen2 5 Text Reasoning", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen2_5_text_reasoning" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "lab_id": "qwen", "modality_class": "vision_language", "name": "Qwen2 5 Vision Language General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen2_5_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "lab_id": "qwen", "modality_class": "vision_language", "name": "Qwen2 5 Vision Language Reasoning", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen2_5_vision_language_reasoning" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_qwen_lineage_qwen2_text_general", "lab_id": "qwen", "modality_class": "text", "name": "Qwen2 Text General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen2_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen2_text_reasoning", "lab_id": "qwen", "modality_class": "text", "name": "Qwen2 Text Reasoning", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen2_text_reasoning" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_qwen_lineage_qwen2_vision_language_general", "lab_id": "qwen", "modality_class": "vision_language", "name": "Qwen2 Vision Language General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen2_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen3_5_text_general", "lab_id": "qwen", "modality_class": "text", "name": "Qwen3 5 Text General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_5_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "lab_id": "qwen", "modality_class": "vision_language", "name": "Qwen3 5 Vision Language General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_5_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen3_6_text_general", "lab_id": "qwen", "modality_class": "text", "name": "Qwen3 6 Text General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_6_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "lab_id": "qwen", "modality_class": "vision_language", "name": "Qwen3 6 Vision Language General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_6_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen3_7_text_general", "lab_id": "qwen", "modality_class": "text", "name": "Qwen3 7 Text General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_7_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "lab_id": "qwen", "modality_class": "vision_language", "name": "Qwen3 7 Vision Language General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_7_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen3_8_text_general", "lab_id": "qwen", "modality_class": "text", "name": "Qwen3 8 Text General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_8_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "lab_id": "qwen", "modality_class": "vision_language", "name": "Qwen3 8 Vision Language General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_8_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen3_text_coding", "lab_id": "qwen", "modality_class": "text", "name": "Qwen3 Text Coding", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_text_coding" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "coding" }, { "id": "lineage_qwen_lineage_qwen3_text_general", "lab_id": "qwen", "modality_class": "text", "name": "Qwen3 Text General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen3_text_reasoning", "lab_id": "qwen", "modality_class": "text", "name": "Qwen3 Text Reasoning", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_text_reasoning" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "reasoning" }, { "id": "lineage_qwen_lineage_qwen3_vision_language_general", "lab_id": "qwen", "modality_class": "vision_language", "name": "Qwen3 Vision Language General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen3_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen_max_text_general", "lab_id": "qwen", "modality_class": "text", "name": "Qwen Max Text General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen_max_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen_text_general", "lab_id": "qwen", "modality_class": "text", "name": "Qwen Text General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen_text_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "lab_id": "qwen", "modality_class": "vision_language", "name": "Qwen Vl Vision Language General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen_vl_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" }, { "id": "lineage_qwen_lineage_qwen_vlo_vision_language_general", "lab_id": "qwen", "modality_class": "vision_language", "name": "Qwen Vlo Vision Language General", "provenance": { "record_ids": [ "lineage_qwen_lineage_qwen_vlo_vision_language_general" ], "record_type": "lineages", "source_file": "data/catalog.yaml" }, "use_class": "general" } ], "models": [ { "id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "name": "Claude 3 model family: Claude 3 Opus, Claude 3 Sonnet, and Claude 3 Haiku", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "name": "Claude 3.5 Sonnet", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "name": "Claude 3.5 Sonnet (New) and Claude 3.5 Haiku", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "name": "Claude 3.7 Sonnet", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "name": "Claude Opus 4 and Claude Sonnet 4", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20250522_introducing_claude_4" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "name": "Claude Opus 4.1", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20250805_claude_opus_4_1" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "name": "Claude Sonnet 4.5", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "name": "Claude Haiku 4.5", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "name": "Claude Opus 4.5", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "name": "Claude Opus 4.6", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "name": "Claude Sonnet 4.6", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "name": "Claude Opus 4.7", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "name": "Claude Opus 4.8", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "name": "Claude Fable 5 and Claude Mythos 5", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "name": "Claude Sonnet 5", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "name": "Claude Opus 5", "provenance": { "record_ids": [ "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_coder_v2_0614", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_coder_v2", "name": "DeepSeek-Coder-V2-0614", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_coder_v2_0614" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_coder_v2_0724", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_coder_v2", "name": "DeepSeek-Coder-V2-0724", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_coder_v2_0724" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_r1", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "name": "DeepSeek-R1", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_r1" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_r1_0528", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "name": "DeepSeek-R1-0528", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_r1_0528" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_r1_lite_preview", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "name": "DeepSeek-R1-Lite-Preview", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_r1_lite_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v2_0517", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "name": "DeepSeek-V2-0517", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v2_0517" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v2_0628", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "name": "DeepSeek-V2-0628", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v2_0628" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v2_5", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "name": "DeepSeek-V2.5", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v2_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v2_5_1210", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "name": "DeepSeek-V2.5-1210", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v2_5_1210" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v3", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "name": "DeepSeek-V3", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v3" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v3_0324", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "name": "DeepSeek-V3-0324", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v3_0324" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v3_1", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "name": "DeepSeek-V3.1", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v3_1" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v3_1_terminus", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "name": "DeepSeek-V3.1-Terminus", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v3_1_terminus" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v3_2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "name": "DeepSeek-V3.2", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v3_2" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v3_2_exp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "name": "DeepSeek-V3.2-Exp", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v3_2_exp" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v3_2_speciale", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "name": "DeepSeek-V3.2-Speciale", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v3_2_speciale" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v4_family", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "name": "DeepSeek-V4 family", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v4_family" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v4_flash_0731", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "name": "DeepSeek-V4-Flash-0731", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v4_flash_0731" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "name": "DeepSeek-V4-Flash-Vision-Exp", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v4_flash_vision_exp" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_deepseek_model_deepseek_v4_pro_0813", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "name": "DeepSeek-V4-Pro-0813", "provenance": { "record_ids": [ "model_deepseek_model_deepseek_v4_pro_0813" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_diffusiongemma", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_diffusiongemma_text_general", "name": "DiffusionGemma", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_diffusiongemma" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 1.5 Flash and Pro update", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_1_5_pro_february_preview", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 1.5 Pro February preview", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_1_5_pro_february_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 1.5 production update", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_1_5_production_update" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 2.0 Flash Experimental", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 2.0 Flash family", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_0_flash_family" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 2.0 Flash native image preview", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 2.0 model family", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_0_model_family" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_5_computer_use", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "name": "Gemini 2.5 Computer Use", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_5_computer_use" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_5_deep_think", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "name": "Gemini 2.5 Deep Think", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_5_deep_think" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 2.5 Flash-Lite", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 2.5 Flash Preview", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 2.5 I/O model family update", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 2.5 Pro Experimental", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 2.5 Pro Preview I/O edition", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 2.5 stable family update", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 3.1 Flash-Lite", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 3.1 Pro", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_3_1_pro" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 3.5 Flash", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_3_5_flash" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_computer_use", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "name": "Gemini 3.5 Flash Computer Use", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_3_5_flash_computer_use" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_cyber", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_cyber_text_coding", "name": "Gemini 3.5 Flash Cyber", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_3_5_flash_cyber" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 3.6 Flash family", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_3_6_flash_family" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 3.7 Flash", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_3_7_flash" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "name": "Gemini 3 Deep Think", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_3_deep_think" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_3_flash", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 3 Flash", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_3_flash" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_3_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini 3 Pro", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_3_pro" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_deep_think_for_science", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "name": "Gemini Deep Think for science", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_deep_think_for_science" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemini_ultra_1_0", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "name": "Gemini Ultra 1.0", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemini_ultra_1_0" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "name": "Gemma 1 release family", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemma_1_release_family" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemma_3_270m", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "name": "Gemma 3 270M", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemma_3_270m" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "name": "Gemma 3 release family", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemma_3_release_family" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemma_3n", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "name": "Gemma 3n", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemma_3n" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "name": "Gemma 3n Preview", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemma_3n_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemma_4_12b", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "name": "Gemma 4 12B", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemma_4_12b" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "name": "Gemma 4 release family", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_gemma_4_release_family" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_medgemma_release_family", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_medgemma_vision_language_general", "name": "MedGemma release family", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_medgemma_release_family" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_t5gemma_release_family", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "name": "T5Gemma release family", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_t5gemma_release_family" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "name": "VaultGemma 1B", "provenance": { "record_ids": [ "model_google_deepmind_google_deepmind_vaultgemma_1b" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_chameleon", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "name": "Meta Chameleon", "provenance": { "record_ids": [ "model_meta_model_meta_chameleon" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_dynamic_blt", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "name": "Meta Dynamic Byte Latent Transformer", "provenance": { "record_ids": [ "model_meta_model_meta_dynamic_blt" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_large_concept_models", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "name": "Meta Large Concept Models", "provenance": { "record_ids": [ "model_meta_model_meta_large_concept_models" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_layerskip_code_llama", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_code", "name": "LayerSkip Code Llama checkpoints", "provenance": { "record_ids": [ "model_meta_model_meta_layerskip_code_llama" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_layerskip_llama", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "name": "LayerSkip Llama checkpoints", "provenance": { "record_ids": [ "model_meta_model_meta_layerskip_llama" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_llama_3", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "name": "Meta Llama 3", "provenance": { "record_ids": [ "model_meta_model_meta_llama_3" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_llama_3_1", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "name": "Meta Llama 3.1", "provenance": { "record_ids": [ "model_meta_model_meta_llama_3_1" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_llama_3_2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "name": "Meta Llama 3.2 collection", "provenance": { "record_ids": [ "model_meta_model_meta_llama_3_2" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_llama_3_2_quantized", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "name": "Quantized Llama 3.2 instruct variants", "provenance": { "record_ids": [ "model_meta_model_meta_llama_3_2_quantized" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_llama_3_3_70b", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "name": "Meta Llama 3.3 70B", "provenance": { "record_ids": [ "model_meta_model_meta_llama_3_3_70b" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_llama_3_3_8b_preview", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "name": "Meta Llama 3.3 8B API preview", "provenance": { "record_ids": [ "model_meta_model_meta_llama_3_3_8b_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_llama_4", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "name": "Meta Llama 4 Scout and Maverick", "provenance": { "record_ids": [ "model_meta_model_meta_llama_4" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_multi_token_prediction", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "name": "Multi-Token Prediction checkpoints", "provenance": { "record_ids": [ "model_meta_model_meta_multi_token_prediction" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_muse_spark", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "name": "Meta Muse Spark", "provenance": { "record_ids": [ "model_meta_model_meta_muse_spark" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_muse_spark_1_1", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "name": "Meta Muse Spark 1.1", "provenance": { "record_ids": [ "model_meta_model_meta_muse_spark_1_1" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_meta_model_meta_plm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "name": "Meta Perception Language Model", "provenance": { "record_ids": [ "model_meta_model_meta_plm" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20240125_new_embedding_models_and_api_updates", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "name": "GPT-4 Turbo dated checkpoints", "provenance": { "record_ids": [ "model_openai_model_openai_20240125_new_embedding_models_and_api_updates" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20240513_hello_gpt_4o", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "name": "GPT-4o", "provenance": { "record_ids": [ "model_openai_model_openai_20240513_hello_gpt_4o" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "name": "GPT-4o mini", "provenance": { "record_ids": [ "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "name": "OpenAI o1-preview", "provenance": { "record_ids": [ "model_openai_model_openai_20240912_introducing_openai_o1_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20240912_openai_o1_mini", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "name": "OpenAI o1-mini", "provenance": { "record_ids": [ "model_openai_model_openai_20240912_openai_o1_mini" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "name": "OpenAI o1 dated snapshot", "provenance": { "record_ids": [ "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20250123_computer_using_agent", "lab_id": "openai", "lineage_id": "lineage_openai_computer_using_agent", "name": "Computer-Using Agent", "provenance": { "record_ids": [ "model_openai_model_openai_20250123_computer_using_agent" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20250131_openai_o3_mini", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "name": "OpenAI o3-mini", "provenance": { "record_ids": [ "model_openai_model_openai_20250131_openai_o3_mini" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "name": "GPT-4.5", "provenance": { "record_ids": [ "model_openai_model_openai_20250227_introducing_gpt_4_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "name": "GPT-4.1 family", "provenance": { "record_ids": [ "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "name": "OpenAI o3 and o4-mini", "provenance": { "record_ids": [ "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20250516_introducing_codex", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "name": "Codex research preview", "provenance": { "record_ids": [ "model_openai_model_openai_20250516_introducing_codex" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20250805_introducing_gpt_oss", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "name": "gpt-oss family", "provenance": { "record_ids": [ "model_openai_model_openai_20250805_introducing_gpt_oss" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20250807_introducing_gpt_5", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "name": "GPT-5 family", "provenance": { "record_ids": [ "model_openai_model_openai_20250807_introducing_gpt_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20250915_introducing_upgrades_to_codex", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "name": "GPT-5-Codex", "provenance": { "record_ids": [ "model_openai_model_openai_20250915_introducing_upgrades_to_codex" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "name": "GPT-5.1 family", "provenance": { "record_ids": [ "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "name": "GPT-5.1-Codex-Max", "provenance": { "record_ids": [ "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "name": "GPT-5.2 family", "provenance": { "record_ids": [ "model_openai_model_openai_20251211_introducing_gpt_5_2" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "name": "GPT-5.2-Codex", "provenance": { "record_ids": [ "model_openai_model_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "name": "GPT-5.3-Codex", "provenance": { "record_ids": [ "model_openai_model_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260212_introducing_gpt_5_3_codex_spark", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "name": "GPT-5.3-Codex-Spark", "provenance": { "record_ids": [ "model_openai_model_openai_20260212_introducing_gpt_5_3_codex_spark" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "name": "GPT-5.3 Instant", "provenance": { "record_ids": [ "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "name": "GPT-5.4 family", "provenance": { "record_ids": [ "model_openai_model_openai_20260305_introducing_gpt_5_4" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "name": "GPT-5.4 mini and nano", "provenance": { "record_ids": [ "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "name": "GPT-Rosalind", "provenance": { "record_ids": [ "model_openai_model_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "name": "GPT-5.5 family", "provenance": { "record_ids": [ "model_openai_model_openai_20260423_introducing_gpt_5_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "name": "GPT-5.5 Instant", "provenance": { "record_ids": [ "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "name": "GPT-Rosalind updated model", "provenance": { "record_ids": [ "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "name": "GPT-5.6 Sol preview", "provenance": { "record_ids": [ "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "name": "GPT-5.6 family", "provenance": { "record_ids": [ "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "name": "GPT-5.6 family price revision", "provenance": { "record_ids": [ "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "name": "GPT-5.6 Sol and Luna update", "provenance": { "record_ids": [ "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_openai_model_openai_20260811_daybreak_models_are_now_available_on_aws", "lab_id": "openai", "lineage_id": "lineage_openai_openai_daybreak", "name": "Daybreak Blue and Red models", "provenance": { "record_ids": [ "model_openai_model_openai_20260811_daybreak_models_are_now_available_on_aws" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "name": "Introducing Qwen", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240123_introducing_qwen_qwen" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "name": "Introducing Qwen-VL", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "name": "Introducing Qwen1.5", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "name": "Qwen1.5-MoE: Matching 7B Model Performance with [activation ratio redacted]", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "name": "Qwen1.5-32B: Fitting the Capstone of the Qwen1.5 Language Model Series", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_coding", "name": "Code with CodeQwen1.5", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "name": "Qwen1.5-110B: The First 100B+ Model of the Qwen1.5 Series", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_max_text_general", "name": "Notes on Qwen-Max-0428", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "name": "Hello Qwen2", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "name": "Introducing Qwen2-Math", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "name": "Qwen2-VL: To See the World More Clearly", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "name": "Qwen2.5: A Party of Foundation Models!", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "name": "Qwen2.5-Coder: Code More, Learn More!", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "name": "Qwen2.5-LLM: Extending the boundary of LLMs", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "name": "Qwen2.5-Math: The world's leading open-sourced mathematical LLMs", "provenance": { "record_ids": [ "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "name": "Qwen2.5-Coder Series: Powerful, Diverse, Practical.", "provenance": { "record_ids": [ "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "name": "Extending the Context Length to 1M Tokens!", "provenance": { "record_ids": [ "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "name": "QwQ: Reflect Deeply on the Boundaries of the Unknown", "provenance": { "record_ids": [ "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "name": "QVQ: To See the World with Wisdom", "provenance": { "record_ids": [ "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "name": "Qwen2.5-1M: Deploy Your Own Qwen with Context Length up to 1M Tokens", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "name": "Qwen2.5 VL! Qwen2.5 VL! Qwen2.5 VL!", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "name": "Qwen2.5-Max: Exploring the Intelligence of Large-scale MoE Model", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "name": "... QwQ-Max-Preview", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "name": "QwQ-32B: Embracing the Power of Reinforcement Learning", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "name": "Qwen2.5-VL-32B: Smarter and Lighter", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "name": "Qwen2.5 Omni: See, Hear, Talk, Write, Do It All!", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "name": "QVQ-Max: Think with Evidence", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "name": "Qwen3: Think Deeper, Act Faster", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vlo_vision_language_general", "name": "Qwen VLo: From \"Understanding\" the World to \"Depicting\" It", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "name": "Qwen3-Coder: Agentic Coding in the World", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "name": "Qwen3-Next: Towards Ultimate Training & Inference Efficiency", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "name": "Qwen3-Omni: Natively Omni-Modal Foundation Models!", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "name": "Qwen3-VL: Sharper Vision, Deeper Thought, Broader Action", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "name": "Qwen3-Max: Just Scale it", "provenance": { "record_ids": [ "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "name": "Qwen3-Omni-Flash-2025-12-01：Hear You. See You. Follow Smarter!", "provenance": { "record_ids": [ "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "name": "Pushing Qwen3-Max-Thinking Beyond its Limits", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260126_qwen3_max_thinking" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "name": "Qwen3-Coder-Next: Pushing Small Hybrid Models on Agentic Coding", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260203_qwen3_coder_next" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260216_qwen3_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "name": "Qwen3.5: Towards Native Multimodal Agents", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260216_qwen3_5" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260319_qwen3_5_max_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_text_general", "name": "Qwen3.5-Max-Preview Now Available on Arena", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260319_qwen3_5_max_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "name": "Qwen3.5-Omni: Scaling Up, Toward Native Omni-Modal AGI", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260330_qwen3_5_omni" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260402_qwen3_6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "name": "Qwen3.6-Plus: Towards Real World Agents", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260402_qwen3_6" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "name": "Qwen3.6-35B-A3B: Agentic Coding Power, Now Open to All", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "name": "Qwen3.6-Max-Preview: Smarter, Sharper, Still Evolving", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260418_qwen3_6_max_preview" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "name": "Qwen3.6-27B: Flagship-Level Coding in a 27B Dense Model", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260422_qwen3_6_27b" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260520_qwen3_7", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "name": "Qwen3.7: The Agent Frontier", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260520_qwen3_7" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "name": "Qwen3.7-Plus: Multimodal Agent Intelligence", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260601_qwen3_7_plus" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260803_qwen3_8", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "name": "Qwen3.8-Max: A New Bar for Coding and Cowork", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260803_qwen3_8" ], "record_type": "models", "source_file": "data/catalog.yaml" } }, { "id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "name": "Qwen3.8-Flash-Next: A New Architecture, Towards Ultimate Cost-Efficiency", "provenance": { "record_ids": [ "model_qwen_model_qwen_20260826_qwen3_8_flash_next" ], "record_type": "models", "source_file": "data/catalog.yaml" } } ], "occurrences": [ { "benchmark_id": "benchmark_ai2d", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multimodal multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "test split; selected images were upsampled while preserving aspect ratio" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_ai2d", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 8, Table 3, AI2D test row; Section" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_ai2d" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_american_mathematics_competition_10", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "repeated answer sampling with official contest scoring" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "official 2023 AMC 10 sets" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_10", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 7, Section, Table 2, AMC 10 row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_10" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_american_mathematics_competition_12", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "repeated answer sampling with official contest scoring" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "official 2023 AMC 12 sets" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_12", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 7, Section, Table 2, AMC 12 row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_12" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_american_mathematics_competition_8", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "repeated answer sampling with official contest scoring" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "official 2023 AMC 8 exam" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_8", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 7, Section, Table 2, AMC 8 row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_american_mathematics_competition_8" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_apps", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "code-execution correctness evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_apps", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, APPS row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_apps" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "challenge split" }, "shot_configuration": { "status": "disclosed", "value": "25-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_arc_challenge", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, ARC-Challenge row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "bias and accuracy calculated from multiple-choice responses" }, "prompting": { "status": "disclosed", "value": "ambiguous and disambiguated multiple-choice question forms" }, "sample_selection": { "status": "disclosed", "value": "U.S. English scenarios spanning multiple social dimensions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bias_benchmark_for_question_answering", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 31, Section, Figure 21" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bias_benchmark_for_question_answering" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "mixed-task benchmark suite" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "BIG-Bench hard task subset" }, "shot_configuration": { "status": "disclosed", "value": "3-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_big_bench_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, BIG-Bench-Hard row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_big_bench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bioasq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice question answering" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bioasq", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 25, Section, automated biology evaluation paragraph" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_bioasq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_chartqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "relaxed answer-matching evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "test split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_chartqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 8, Table 3, ChartQA test row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "answer-similarity evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "test split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_docvqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 8, Table 3, DocVQA test row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "token-level answer comparison" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "3-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_drop", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, DROP row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_easy_medium_qa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "answers classified against reference answers" }, "prompting": { "status": "disclosed", "value": "closed-ended factual questions" }, "sample_selection": { "status": "disclosed", "value": "handwritten questions selected to test factual knowledge and excessive uncertainty" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_easy_medium_qa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 18, Section, Easy-Medium QA bullet" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_easy_medium_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "repeated rollouts with randomized answer-option order; majority voting for the sampled variant" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "GPQA Diamond subset" }, "shot_configuration": { "status": "disclosed", "value": "0-shot and 5-shot; a majority-vote variant was also reported" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, GPQA (Diamond) row; Appendix C, Table 8" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gre_general_test_analytical_writing", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official practice-test scoring" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "official GRE practice test, with a separate official test supplying examples" }, "shot_configuration": { "status": "disclosed", "value": "2-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_analytical_writing", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 7, Section, Table 2, GRE (Writing) row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_analytical_writing" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gre_general_test_quantitative_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official practice-test scoring" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "official GRE practice test, with a separate official test supplying examples" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_quantitative_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 7, Section, Table 2, GRE (Quantitative) row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_quantitative_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gre_general_test_verbal_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official practice-test scoring" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "official GRE practice test, with a separate official test supplying examples" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_verbal_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 7, Section, Table 2, GRE (Verbal) row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gre_general_test_verbal_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gsm8k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, GSM8K row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice completion evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "10-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_hellaswag", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, HellaSwag row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "code-generation correctness evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_humaneval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, HumanEval row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_law_school_admission_test", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "scaled-score aggregation across official practice tests" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "official practice tests, with separate official tests supplying examples" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_law_school_admission_test", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 7, Section, Table 2, LSAT row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_law_school_admission_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multimodal multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "validation split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 8, Table 3, MMMU validation row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice evaluation" }, "prompting": { "status": "disclosed", "value": "direct and chain-of-thought variants" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, MMLU row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_massive_multitask_language_understanding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "direct-answer and repeated-sampling majority-vote variants" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot and 4-shot; a majority-vote variant was also reported" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, MATH row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multimodal question-answering evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "testmini split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mathvista", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 8, Table 3, MathVista testmini row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_medmcqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice question answering" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_medmcqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 25, Section, automated biology evaluation paragraph" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_medmcqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single submitted generation checked for correctness" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mostly_basic_python_problems", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, MBPP row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_mostly_basic_python_problems" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_multi_factual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "answers classified as correct, incorrect, or unsure against references" }, "prompting": { "status": "disclosed", "value": "open-ended multi-part factual questions" }, "sample_selection": { "status": "disclosed", "value": "questions synthesized from article quotations and manually verified" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multi_factual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 18, Section, Multi-factual bullet and results discussion" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multi_factual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multilingual question-answering evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "translated GSM8K problems across multiple languages" }, "shot_configuration": { "status": "disclosed", "value": "0-shot and 8-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multilingual_grade_school_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 16, Section, Table 4, MGSM row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multilingual_grade_school_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multilingual multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "multilingual MMLU version" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multilingual_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 16, Section, Table 5" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multilingual_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_multistate_bar_examination", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official practice-exam evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "official MBE practice exam" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multistate_bar_examination", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 7, Section, Table 2, MBE row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_multistate_bar_examination" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_needle_in_a_haystack", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "recall measured across varying context lengths and insertion positions" }, "prompting": { "status": "disclosed", "value": "modified retrieval prompt" }, "sample_selection": { "status": "disclosed", "value": "random needle-question pairs and both essay-based and crowdsourced document corpora" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_needle_in_a_haystack", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 21, Section; Figure 15 and Table 7" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_needle_in_a_haystack" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_pubmedqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice biomedical question answering" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot and 5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_pubmedqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, PubMedQA row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_pubmedqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_quality", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice question answering with sampled responses" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "long stories adversarially selected to resist time-limited skimming" }, "shot_configuration": { "status": "disclosed", "value": "0-shot and 1-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_quality", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 20, Section; Table 6" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_quality" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_race_high", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice reading comprehension" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "RACE high-school subset" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_race_high", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, RACE-H row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_race_high" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_united_states_medical_licensing_examination", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice question answering" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_united_states_medical_licensing_examination", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 25, Section, automated biology evaluation paragraph" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_united_states_medical_licensing_examination" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_wildchat", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "model-evaluated refusal classification" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "toxic and non-toxic WildChat subsets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_wildchat", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 10, Section; Figure 2" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_wildchat" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_winogrande", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 6, Table 1, WinoGrande row" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_record_100q_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "answers classified as correct, incorrect, or unsure against references" }, "prompting": { "status": "disclosed", "value": "open-ended factual questions" }, "sample_selection": { "status": "disclosed", "value": "human-written questions selected for obscurity and prior-model difficulty" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_x_100q_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 18, Section, 100Q Hard bullet and Figure 11 discussion" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_x_100q_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_xstest", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "incorrect-refusal classification" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "non-malicious XSTest prompts" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_xstest", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "locator": { "kind": "page", "value": "page 10, Section; Figure 3" }, "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240304_introducing_the_next_generation_of_claude_benchmark_xstest" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "release_id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "review_status": "verified", "source_id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_revision_id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_ai2d", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multimodal question-answering evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "test split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_ai2d", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 2, AI2D test row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_ai2d" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "mixed-task benchmark suite" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "BIG-Bench hard task subset" }, "shot_configuration": { "status": "disclosed", "value": "3-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_big_bench_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 1, BIG-Bench Hard row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_big_bench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_chartqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "relaxed answer-matching evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "test split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_chartqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 2, ChartQA test row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "answer-similarity evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "test split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_docvqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 2, DocVQA test row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "token-level answer comparison" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "3-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_drop", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 1, DROP row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "direct and repeated-sampling majority-vote variants" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "GPQA Diamond subset" }, "shot_configuration": { "status": "disclosed", "value": "0-shot and 5-shot; a majority-vote variant was also reported" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 1, GPQA (Diamond) row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_gsm8k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 1, GSM8K row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "code-generation correctness evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_humaneval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 1, HumanEval row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multimodal question-answering evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "validation split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 2, MMMU validation row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice evaluation" }, "prompting": { "status": "disclosed", "value": "direct and chain-of-thought variants" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot and 5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 1, MMLU row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_massive_multitask_language_understanding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 1, MATH row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multimodal question-answering evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "testmini split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_mathvista", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 2, MathVista testmini row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multilingual question-answering evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_multilingual_grade_school_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 2, Table 1, MGSM row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_multilingual_grade_school_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_needle_in_a_haystack", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "recall measured across context lengths" }, "prompting": { "status": "disclosed", "value": "modified retrieval prompt" }, "sample_selection": { "status": "disclosed", "value": "Anthropic's modified Needle In A Haystack task" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_needle_in_a_haystack", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 3, Needle In A Haystack section, Table 5; Figures 1 and 2" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_needle_in_a_haystack" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_wildchat", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "correct-refusal and incorrect-refusal classification" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "toxic and non-toxic WildChat subsets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_wildchat", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 3, Refusals section, Table 4, Wildchat Toxic and Wildchat Non-toxic rows" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_wildchat" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_xstest", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "incorrect-refusal classification" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "XSTest prompts" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_xstest", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 3, Refusals section, Table 4, XSTest row" }, "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20240621_claude_3_5_sonnet_benchmark_xstest" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "release_id": "release_anthropic_20240621_claude_3_5_sonnet", "review_status": "verified", "source_id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_revision_id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_ai2d", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multimodal question-answering evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "test split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_ai2d", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 5, Section, Table 5, AI2D test row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_ai2d" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "direct and majority-vote answer evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "questions from both official 2024 tests" }, "shot_configuration": { "status": "disclosed", "value": "0-shot; a repeated-sampling majority-vote variant was also reported" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_american_invitational_mathematics_examination_2024", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 6, Section and Table 7, AIME 2024 row; Table 8, AIME 2024 row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_american_invitational_mathematics_examination_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multi-trial reliability evaluation over user-agent-tool trajectories" }, "prompting": { "status": "disclosed", "value": "role, policy, and scenario instructions" }, "sample_selection": { "status": "disclosed", "value": "retail and airline domain tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "simulated customer-service APIs" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 3, Section; Table 3 and Figure 1" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "mixed-task benchmark suite" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "BIG-Bench hard task subset" }, "shot_configuration": { "status": "disclosed", "value": "3-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_big_bench_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 6, Table 7, BIG-Bench Hard row; Table 8, BIG-Bench Hard row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_big_bench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_chartqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "relaxed answer-matching evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "test split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_chartqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 5, Section, Table 5, ChartQA test row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "answer-similarity evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "test split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_docvqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 5, Section, Table 5, DocVQA test row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "token-level answer comparison" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "3-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_drop", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 6, Table 7, DROP row; Table 8, DROP row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "GPQA Diamond subset" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 6, Table 7, GPQA (Diamond) row; Table 8, GPQA (Diamond) row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "code-generation correctness evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_humaneval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 6, Table 7, HumanEval row; Table 8, HumanEval row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "instruction-constraint compliance evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specified detailed instructions" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_instruction_following_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 6, Section and Table 7, IFEval row; Table 8, IFEval row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_instruction_following_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multimodal question-answering evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "validation split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 5, Section, Table 5, MMMU validation row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice evaluation" }, "prompting": { "status": "disclosed", "value": "direct and chain-of-thought variants" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot and 5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 6, Table 7, MMLU row; Table 8, MMLU row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_massive_multitask_language_understanding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 6, Table 7, MATH row; Table 8, MATH row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multimodal question-answering evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "testmini split" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mathvista", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 5, Section, Table 5, MathVista testmini row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "disclosed", "value": "MMLU Pro" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mmlu_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 6, Table 7, MMLU Pro row; Table 8, MMLU Pro row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multilingual question-answering evaluation" }, "prompting": { "status": "disclosed", "value": "chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_multilingual_grade_school_math", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 6, Table 7, MGSM row; Table 8, MGSM row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_multilingual_grade_school_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OSWorld setup configurations and evaluation scripts with standard and extended interaction budgets" }, "prompting": { "status": "disclosed", "value": "optimized computer-use prompt" }, "sample_selection": { "status": "disclosed", "value": "screenshot-only input condition" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "GUI computer commands generated from screenshots" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 2, Section, Table 1" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_osworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "hidden grading tests from the issue-resolving pull requests" }, "prompting": { "status": "disclosed", "value": "dedicated agentic prompt; further prompt optimization was identified as possible" }, "sample_selection": { "status": "disclosed", "value": "SWE-bench Verified tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository search, code viewing, editing, and test execution in an iterative loop" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 3, Section, Table 2" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_wildchat", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "correct-refusal and incorrect-refusal classification" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "toxic and non-toxic WildChat subsets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_wildchat", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 5, Section, Table 6, Wildchat Toxic and Wildchat Non-toxic rows" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_wildchat" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_xstest", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "incorrect-refusal classification" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "XSTest prompts" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_xstest", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "locator": { "kind": "page", "value": "page 5, Section, Table 6, XSTest row" }, "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and_benchmark_xstest" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "release_id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "review_status": "verified", "source_id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_revision_id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "source_type": "model_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multi-turn trajectories with an increased completion-step allowance" }, "prompting": { "status": "disclosed", "value": "airline-policy addendum encouraging use of a planning tool" }, "sample_selection": { "status": "disclosed", "value": "updated TAU-bench dataset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "benchmark APIs, simulated-user interaction, and a planning tool" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "heading", "value": "Claude Sonnet: Frontier reasoning made practical; Appendix, TAU-bench, Information about the scaffolding" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "bias and accuracy evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark multiple-choice questions" }, "sample_selection": { "status": "disclosed", "value": "ambiguous and disambiguated BBQ questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bias_benchmark_for_question_answering", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 12, Section; Tables 1 and 2" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bias_benchmark_for_question_answering" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bioweapons_knowledge_questions", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "manual grading by external experts using question-specific rubrics" }, "prompting": { "status": "disclosed", "value": "open-ended domain-specific questions" }, "sample_selection": { "status": "disclosed", "value": "expert-developed biological-weapons knowledge questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bioweapons_knowledge_questions", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 25, Table 5; Bioweapons Knowledge Questions details and Figure 17" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_bioweapons_knowledge_questions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "repeated challenge trials with binary task success" }, "prompting": { "status": "disclosed", "value": "challenge-specific capture-the-flag objectives" }, "sample_selection": { "status": "disclosed", "value": "implemented subset of Cybench challenges" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "cybersecurity challenge environment" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 35, Section, Table 7; Cybench details; Figures 25 and 26" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_cybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cyber_harness_network_challenges", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "success required complete access and extraction of specified data across hosts" }, "prompting": { "status": "disclosed", "value": "autonomous network-attack objectives" }, "sample_selection": { "status": "disclosed", "value": "easy and medium cyber-range challenges" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "specialized offensive cyber harness" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_cyber_harness_network_challenges", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 37, Section, Table 7; Cyber-harness network challenges details and Figure 24" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_cyber_harness_network_challenges" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "comparison of answer changes and explicit clue acknowledgment in chain-of-thought" }, "prompting": { "status": "disclosed", "value": "constructed prompt pairs containing neutral or misaligned clues" }, "sample_selection": { "status": "disclosed", "value": "multiple-choice GPQA questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_gpqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 17, Section; Figure 12" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "environment-specific speed, quality, or completion measures against expert references" }, "prompting": { "status": "disclosed", "value": "environment-specific machine-learning optimization tasks" }, "sample_selection": { "status": "disclosed", "value": "internal environments spanning several machine-learning problem types" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic coding and model-training environments" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_internal_ai_research_evaluation_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 30, Section, Table 6; Internal AI Research Evaluation Suite details; Figure 22" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_internal_ai_research_evaluation_suite" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_labbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "human-baselined evaluation across selected LabBench tasks" }, "prompting": { "status": "disclosed", "value": "multiple-choice questions" }, "sample_selection": { "status": "disclosed", "value": "FigQA, ProtocolQA, SeqQA, and CloningScenarios" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search and bioinformatics tools were not provided" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 25, Table 5; LabBench Subset details and Figure 18; Figure 19" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_labbench_cloningscenarios", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LabBench multiple-choice evaluation with human baselines" }, "prompting": { "status": "disclosed", "value": "multiple-choice cloning-workflow questions" }, "sample_selection": { "status": "disclosed", "value": "CloningScenarios task" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search and bioinformatics tools were not provided" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 28, LabBench Subset details and Figure 18" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_cloningscenarios" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_labbench_figqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LabBench multiple-choice evaluation with human baselines" }, "prompting": { "status": "disclosed", "value": "multiple-choice figure questions" }, "sample_selection": { "status": "disclosed", "value": "FigQA task" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search and bioinformatics tools were not provided" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 28, LabBench Subset details and Figure 18" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_figqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_labbench_protocolqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LabBench multiple-choice evaluation with human baselines" }, "prompting": { "status": "disclosed", "value": "multiple-choice protocol questions" }, "sample_selection": { "status": "disclosed", "value": "ProtocolQA task" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search and bioinformatics tools were not provided" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 28, LabBench Subset details and Figure 18; Figure 19" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_protocolqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_labbench_seqqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LabBench multiple-choice evaluation with human baselines" }, "prompting": { "status": "disclosed", "value": "multiple-choice sequence questions" }, "sample_selection": { "status": "disclosed", "value": "SeqQA task" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search and bioinformatics tools were not provided" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 28, LabBench Subset details and Figure 18" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_labbench_seqqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_long_form_virology_tasks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "completion assessed across workflow-design and laboratory-protocol subtasks" }, "prompting": { "status": "disclosed", "value": "long-form task instructions" }, "sample_selection": { "status": "disclosed", "value": "task set developed with external biological-risk specialists" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic task environment; specific tools not disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_long_form_virology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 25, Table 5; Long-form Virology Tasks details" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_long_form_virology_tasks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "comparison of answer changes and explicit clue acknowledgment in chain-of-thought" }, "prompting": { "status": "disclosed", "value": "constructed prompt pairs containing neutral or misaligned clues" }, "sample_selection": { "status": "disclosed", "value": "multiple-choice MMLU questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_massive_multitask_language_understanding", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 17, Section; Figure 12" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_massive_multitask_language_understanding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_metr_data_deduplication", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "output quality evaluated against a deduplication threshold" }, "prompting": { "status": "disclosed", "value": "task specification for data deduplication" }, "sample_selection": { "status": "disclosed", "value": "METR data-deduplication task" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic coding environment" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_metr_data_deduplication", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 30, Section, Table 6 and METR Data Deduplication details; Figure 20" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_metr_data_deduplication" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_re_bench_subset", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "modified internal infrastructure with reference and expert solutions" }, "prompting": { "status": "disclosed", "value": "task-specific research-engineering instructions" }, "sample_selection": { "status": "disclosed", "value": "selected RE-Bench tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic research-engineering environment" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_re_bench_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 30, Section, Table 6; RE-Bench Subset details and Figure 23" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_re_bench_subset" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_secure_bio_lab_assistance_benchmark_multimodal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "an item was correct only when every correct option and no incorrect option was selected" }, "prompting": { "status": "disclosed", "value": "multiple-select multimodal questions" }, "sample_selection": { "status": "disclosed", "value": "virology-specific questions shared among participating laboratories" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_secure_bio_lab_assistance_benchmark_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 25, Table 5; Multimodal Virology details; Figure 16" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_secure_bio_lab_assistance_benchmark_multimodal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-session scaffold; a separate multi-attempt variant filtered regressions and selected a final patch" }, "prompting": { "status": "disclosed", "value": "agentic coding prompt with a planning-tool addendum" }, "sample_selection": { "status": "disclosed", "value": "official verified tasks, with infrastructure-incompatible cases retained as failures for leaderboard parity and a compatible-task subset reported separately" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash, file editing, and planning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "heading", "value": "Claude Sonnet: Frontier reasoning made practical; Appendix, SWE-bench Verified, Information about the scaffolding" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "unit-test grader with an expert reference patch" }, "prompting": { "status": "disclosed", "value": "original issue description and relevant code context" }, "sample_selection": { "status": "disclosed", "value": "valid tasks estimated to require substantial human software-engineering time" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic repository modification" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "locator": { "kind": "page", "value": "page 30, Section, Table 6 and SWE-bench Verified (hard subset) details" }, "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250224_claude_3_7_sonnet_and_claude_code_benchmark_swe_bench_verified_hard_subset" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "release_id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "review_status": "verified", "source_id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_revision_id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aime", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "extended-thinking configuration" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "locator": { "kind": "heading", "value": "Appendix, Performance benchmark reporting, AIME entry" }, "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_aime" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250522_introducing_claude_4", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250522_introducing_claude_4", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multi-turn airline and retail agent-policy trajectories with an increased completion-step allowance" }, "prompting": { "status": "disclosed", "value": "prompt addenda to the Airline and Retail Agent Policy encouraged explicit reasoning during trajectories" }, "sample_selection": { "status": "disclosed", "value": "Airline and Retail tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "extended thinking with tool use" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "locator": { "kind": "heading", "value": "Appendix, Performance benchmark reporting, TAU-bench entry; TAU-bench methodology" }, "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250522_introducing_claude_4", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250522_introducing_claude_4", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "extended-thinking configuration" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "GPQA Diamond subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "locator": { "kind": "heading", "value": "Appendix, Performance benchmark reporting, GPQA Diamond entry" }, "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250522_introducing_claude_4", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250522_introducing_claude_4", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "extended-thinking configuration" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "locator": { "kind": "heading", "value": "Appendix, Performance benchmark reporting, MMMLU entry" }, "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250522_introducing_claude_4", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250522_introducing_claude_4", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "extended-thinking configuration" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_mmmu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "locator": { "kind": "heading", "value": "Appendix, Performance benchmark reporting, MMMU entry" }, "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250522_introducing_claude_4", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250522_introducing_claude_4", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "simple scaffold without the former planning tool; a separate high-compute configuration used parallel attempts, visible-test filtering, and internal candidate selection" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "full SWE-bench Verified problem set for Anthropic models" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash and string-replacement file-editing tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "locator": { "kind": "heading", "value": "Claude 4, SWE-bench Verified benchmark row; Appendix, SWE-bench methodology" }, "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250522_introducing_claude_4", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250522_introducing_claude_4", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "locator": { "kind": "heading", "value": "Claude 4, Terminal-bench benchmark row; Appendix, Performance benchmark reporting" }, "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250522_introducing_claude_4_benchmark_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "release_id": "release_anthropic_20250522_introducing_claude_4", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250522_introducing_claude_4", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250522_introducing_claude_4", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aime", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "extended-thinking configuration" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Appendix, Benchmark reporting, extended-thinking list, AIME entry" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_aime" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250805_claude_opus_4_1", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250805_claude_opus_4_1", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multi-turn airline and retail agent-policy trajectories with an increased completion-step allowance" }, "prompting": { "status": "disclosed", "value": "prompt addenda to the Airline and Retail Agent Policy encouraged explicit reasoning during trajectories" }, "sample_selection": { "status": "disclosed", "value": "Airline and Retail tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "extended thinking with tool use" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Appendix, Benchmark reporting, extended-thinking list; TAU-bench methodology" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250805_claude_opus_4_1", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250805_claude_opus_4_1", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "standard non-extended-thinking mode" }, "prompting": { "status": "disclosed", "value": "standard benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions spanning multiple social dimensions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_bias_benchmark_for_question_answering", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 7, PDF, section, Tables .A and .B" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_bias_benchmark_for_question_answering" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cloningscenarios", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Cloning Scenarios subtask" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 21, PDF, section, LAB-Bench subset bullet, Cloning Scenarios task" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_cloningscenarios" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_creative_biology", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "creative biology question set" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 20, PDF, section, Figure .B" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_creative_biology" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_figqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "FigQA subtask" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 21, PDF, section, LAB-Bench subset bullet, FigQA task" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_figqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "extended-thinking configuration" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "GPQA Diamond subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Appendix, Benchmark reporting, extended-thinking list, GPQA Diamond entry" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250805_claude_opus_4_1", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250805_claude_opus_4_1", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal AI-research agent suite" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "non-saturated evaluations only" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-dependent coding and research tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, PDF, section, AI research capabilities list" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_internal_ai_research_evaluation_suite_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_kernel_optimization", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal machine-learning kernel optimization task" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "hard task variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_kernel_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, PDF, section, Kernel optimization bullet" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_kernel_optimization" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multiple-choice biological-research evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "ProtocolQA, SeqQA, Cloning Scenarios, and FigQA subtasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 21, PDF, section, LAB-Bench subset bullet" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_lab_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_llm_training_optimization", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal language-model training optimization task" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding and machine-learning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_llm_training_optimization", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, PDF, section, LLM training optimization bullet" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_llm_training_optimization" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_long_form_virology_task_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "long-form task with separate sequence-design and protocol-design components" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Long-Form Virology Task 1" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic task tools not further disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 21, PDF, section, Long-Form Virology Task 1 bullet" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_long_form_virology_task_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_long_form_virology_task_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "long-form task with separate sequence-design and protocol-design components" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Long-Form Virology Task 2" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic task tools not further disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 21, PDF, section, Long-Form Virology Task 2 bullet" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_long_form_virology_task_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "extended-thinking configuration" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Appendix, Benchmark reporting, extended-thinking list, MMMLU entry" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250805_claude_opus_4_1", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250805_claude_opus_4_1", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "extended-thinking configuration" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_mmmu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Appendix, Benchmark reporting, extended-thinking list, MMMU entry" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250805_claude_opus_4_1", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250805_claude_opus_4_1", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_novel_compiler", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal compiler-construction task with basic and advanced tests" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, PDF, section, Novel compiler bullet" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_novel_compiler" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "ProtocolQA subtask" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 21, PDF, section, LAB-Bench subset bullet, ProtocolQA task" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_protocolqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_quadruped_locomotion", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal quadruped locomotion task" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "easier task variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding and machine-learning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_quadruped_locomotion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, PDF, section, Quadruped locomotion bullet" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_quadruped_locomotion" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_seqqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "SeqQA subtask" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 21, PDF, section, LAB-Bench subset bullet, SeqQA task" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_seqqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "short-horizon computational biology task suite" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "non-saturated computational biology tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bioinformatics tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 19, PDF, section, Short-horizon computational biology tasks bullet and Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_short_horizon_computational_biology_tasks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "simple Claude 4 scaffold without the former planning tool" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "full SWE-bench Verified problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash and string-replacement file-editing tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Claude Opus, SWE-bench Verified result; Appendix, SWE-bench methodology" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250805_claude_opus_4_1", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250805_claude_opus_4_1", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "autonomy checkpoint evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "hard SWE-bench Verified subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-engineering agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, PDF, section, Software engineering capabilities bullet" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_swe_bench_verified_hard_subset" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_synthesis_screening_evasion", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "criterion-based synthesis-screening-evasion evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 21, PDF, section, Synthesis Screening Evasion bullet and Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_synthesis_screening_evasion" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Appendix, Benchmark reporting, No extended thinking list, Terminal-Bench entry" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250805_claude_opus_4_1", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250805_claude_opus_4_1", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_text_based_reinforcement_learning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal text-based reinforcement-learning task" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding and machine-learning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_text_based_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, PDF, section, Text-based RL bullet" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_text_based_reinforcement_learning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_time_series_forecasting", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal time-series forecasting task" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "hard task variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding and machine-learning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, PDF, section, Time series forecasting bullet" }, "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250805_claude_opus_4_1_benchmark_time_series_forecasting" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_anthropic_20250805_claude_opus_4_1", "review_status": "verified", "source_id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_revision_id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_agent_red_teaming", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external Gray Swan agent red-teaming framework with extended-thinking and standard variants" }, "prompting": { "status": "disclosed", "value": "adversarial instructions targeting confidentiality, safety, malicious activity, and unauthorized tool use" }, "sample_selection": { "status": "disclosed", "value": "ART prompt-injection attack categories" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_agent_red_teaming", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 29, PDF, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_agent_red_teaming" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_agentic_misalignment_suite", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "expanded hand-built suite run with and without extended thinking" }, "prompting": { "status": "disclosed", "value": "expert-written simulated organizational scenarios" }, "sample_selection": { "status": "disclosed", "value": "blackmail, research-sabotage, and financial-crime framing scenarios" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "scenario-dependent agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_agentic_misalignment_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 72, PDF, section, Agentic misalignment suites" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_agentic_misalignment_suite" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aime", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "sampling configuration with extended reasoning" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Python configuration" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "heading", "value": "Methodology, AIME" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_aime" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "standard non-extended-thinking mode" }, "prompting": { "status": "disclosed", "value": "standard ambiguous and disambiguated benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "ambiguous and disambiguated question sets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_bias_benchmark_for_question_answering", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 19, PDF, section, Tables .A and .B" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_bias_benchmark_for_question_answering" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_blackmail_suite", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "behavioral and white-box interpretability analyses" }, "prompting": { "status": "disclosed", "value": "simulated blackmail scenarios" }, "sample_selection": { "status": "disclosed", "value": "Blackmail subset of the Agentic Misalignment suite" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "scenario-dependent agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_blackmail_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 90, PDF, section, Datasets used for evaluation, item Blackmail Suite" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_blackmail_suite" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cloningscenarios", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "CloningScenarios subtask" }, "shot_configuration": { "status": "disclosed", "value": "ten-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 133, PDF, section, CloningScenarios subtask and Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cloningscenarios" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_computer_use_prompt_injection_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "virtual-machine tasks run with and without additional safeguards" }, "prompting": { "status": "disclosed", "value": "adversarial instructions were embedded in compromised files or websites" }, "sample_selection": { "status": "disclosed", "value": "hand-built computer-use prompt-injection cases" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "standard computer actions in a virtual machine" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_computer_use_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 30, PDF, section, Computer use evaluation table" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_computer_use_prompt_injection_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_creative_biology", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "expert-oriented question set designed to limit information hazards" }, "prompting": { "status": "disclosed", "value": "creative biological-engineering questions" }, "sample_selection": { "status": "disclosed", "value": "SecureBio creative biology questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_creative_biology", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 134, PDF, section, Creative biology" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_creative_biology" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "capture-the-flag agent evaluation grouped by challenge category and difficulty" }, "prompting": { "status": "disclosed", "value": "challenge instructions" }, "sample_selection": { "status": "disclosed", "value": "infrastructure-compatible subset of original Cybench challenges" }, "shot_configuration": { "status": "disclosed", "value": "multiple-attempt configurations" }, "tool_use": { "status": "disclosed", "value": "cybersecurity challenge tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 37, PDF, section, Figures .A and .B" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "custom scaffold with context reset and autosummarization; both cost-constrained and unconstrained configurations" }, "prompting": { "status": "disclosed", "value": "textual vulnerability descriptions plus source code" }, "sample_selection": { "status": "disclosed", "value": "historical real-world vulnerabilities and post-patch code" }, "shot_configuration": { "status": "disclosed", "value": "single-attempt and repeated-attempt configurations" }, "tool_use": { "status": "disclosed", "value": "codebase and execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 35, PDF, section, Figures .A-.C" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_dna_synthesis_screening_evasion", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "criterion-based assembly-viability and screening-evasion evaluation" }, "prompting": { "status": "disclosed", "value": "DNA fragment design task" }, "sample_selection": { "status": "disclosed", "value": "pathogenic-agent sequences of varied lengths" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_dna_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 132, PDF, section, DNA Synthesis Screening Evasion" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_dna_synthesis_screening_evasion" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_false_premise_questions", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "self-consistency evaluation across paired question forms" }, "prompting": { "status": "disclosed", "value": "paired questions that presupposed or directly queried the same false premise" }, "sample_selection": { "status": "disclosed", "value": "questions about fabricated concepts and claims" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_false_premise_questions", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 23, PDF, section, False-Premise Questions" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_false_premise_questions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_figqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "FigQA subtask" }, "shot_configuration": { "status": "disclosed", "value": "ten-shot" }, "tool_use": { "status": "disclosed", "value": "image input" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 133, PDF, section, FigQA subtask and Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_figqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_financial_crimes", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "behavioral and white-box interpretability analyses" }, "prompting": { "status": "disclosed", "value": "financial-crime scenarios from the Agentic Misalignment evaluation" }, "sample_selection": { "status": "disclosed", "value": "Financial Crimes evaluation set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "scenario-dependent agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_financial_crimes", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 90, PDF, section, Datasets used for evaluation, item Financial Crimes" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_financial_crimes" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_impossible_tasks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "controlled scaffold outside Claude Code" }, "prompting": { "status": "disclosed", "value": "minimal-instruction and explicit anti-hardcoding variants" }, "sample_selection": { "status": "disclosed", "value": "problems intentionally designed to be impossible" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_impossible_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 49, PDF, section, Table .A and Impossible Tasks description" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_impossible_tasks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_incalmo_cyber_ranges", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multi-host cyber ranges measuring asset acquisition and attack progression" }, "prompting": { "status": "disclosed", "value": "network-attack objectives" }, "sample_selection": { "status": "disclosed", "value": "academic-collaboration cyber ranges" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "network and cyber-operation tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_incalmo_cyber_ranges", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 43, PDF, section, Incalmo cyber ranges" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_incalmo_cyber_ranges" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal agent environments covering model training, forecasting, optimization, reinforcement learning, and compiler work" }, "prompting": { "status": "disclosed", "value": "task-specific machine-learning improvement objectives" }, "sample_selection": { "status": "disclosed", "value": "suite tasks and disclosed difficulty variants" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding and machine-learning experimentation tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_internal_ai_research_evaluation_suite_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 139, PDF, section" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_internal_ai_research_evaluation_suite_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_internal_ai_research_evaluation_suite_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "automated internal research suite with outcome-based and partial-credit grading" }, "prompting": { "status": "disclosed", "value": "task-specific research objectives" }, "sample_selection": { "status": "disclosed", "value": "tasks based on research training materials and scaled-down internal projects" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "research, coding, and machine-learning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_internal_ai_research_evaluation_suite_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 139, PDF, sections and" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_internal_ai_research_evaluation_suite_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_kernels_task", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "iterative kernel optimization environment" }, "prompting": { "status": "disclosed", "value": "kernel-improvement objective" }, "sample_selection": { "status": "disclosed", "value": "disclosed task variants" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding and benchmarking tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_kernels_task", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 141, PDF, section, Kernels task" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_kernels_task" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "FutureHouse multiple-choice assessment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "FigQA, ProtocolQA, SeqQA, and CloningScenarios" }, "shot_configuration": { "status": "disclosed", "value": "ten-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 133, PDF, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_lab_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_llm_training", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal language-model training environment" }, "prompting": { "status": "disclosed", "value": "model-training optimization objective" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding and machine-learning training tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_llm_training", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 144, PDF, section, LLM training" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_llm_training" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_long_form_virology_task_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "long-form task with molecular sequence and laboratory protocol components" }, "prompting": { "status": "disclosed", "value": "helpful-only and safety-trained model variants were elicited, with the strongest disclosed result reported" }, "sample_selection": { "status": "disclosed", "value": "Long-Form Virology Task 1" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search, research, and agentic task tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 130, PDF, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_long_form_virology_task_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_long_form_virology_task_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "long-form task with molecular sequence and laboratory protocol components" }, "prompting": { "status": "disclosed", "value": "helpful-only and safety-trained model variants were elicited, with the strongest disclosed result reported" }, "sample_selection": { "status": "disclosed", "value": "Long-Form Virology Task 2" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search, research, and agentic task tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 131, PDF, section, Figure .B" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_long_form_virology_task_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mcp_prompt_injection_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "routine agent tasks containing compromised content, run with and without a detection classifier" }, "prompting": { "status": "disclosed", "value": "embedded adversarial instructions attempted to redirect routine assigned work" }, "sample_selection": { "status": "disclosed", "value": "hand-built MCP prompt-injection cases" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "simulated email, messaging, and document-collaboration MCP tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mcp_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 30, PDF, section, Model Context Protocol evaluation table" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mcp_prompt_injection_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "extended-thinking configuration averaged across repeated runs" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "non-English language subsets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "heading", "value": "Methodology, MMMLU" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_novel_compiler", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal compiler-construction environment with basic and advanced tests" }, "prompting": { "status": "disclosed", "value": "formal language specification and compiler objective" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "no external examples disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_novel_compiler", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 146, PDF, section, Novel compiler" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_novel_compiler" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_opinion_elicitation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "white-box interpretability analyses" }, "prompting": { "status": "disclosed", "value": "handwritten opinion questions" }, "sample_selection": { "status": "disclosed", "value": "questions about the model, humans, and society" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "none disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_opinion_elicitation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 90, PDF, section, Datasets used for evaluation, item Opinion Elicitation" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_opinion_elicitation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official OSWorld-Verified framework with a bounded-step agent and repeated runs" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "OSWorld tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "computer interaction" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "heading", "value": "Frontier intelligence, OSWorld result; Methodology, OSWorld" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_osworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_pretraining_documents", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "white-box interpretability control analysis" }, "prompting": { "status": "disclosed", "value": "pretraining-style documents" }, "sample_selection": { "status": "disclosed", "value": "randomly selected pretraining documents" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "none" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_pretraining_documents", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 90, PDF, section, Datasets used for evaluation, item Pretraining Documents" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_pretraining_documents" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "ProtocolQA subtask" }, "shot_configuration": { "status": "disclosed", "value": "ten-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 133, PDF, section, ProtocolQA subtask and Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_protocolqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_quadruped_reinforcement_learning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal embodied reinforcement-learning environment" }, "prompting": { "status": "disclosed", "value": "locomotion-training objective" }, "sample_selection": { "status": "disclosed", "value": "easy task variants reported" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding, simulation, and machine-learning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_quadruped_reinforcement_learning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 145, PDF, section, Quadruped reinforcement learning" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_quadruped_reinforcement_learning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_random_chat_questions", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "white-box interpretability control analysis" }, "prompting": { "status": "disclosed", "value": "generic chat questions" }, "sample_selection": { "status": "disclosed", "value": "model-generated generic chat questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "none" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_random_chat_questions", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 90, PDF, section, Datasets used for evaluation, item Random Chat Questions" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_random_chat_questions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_research_sabotage", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "behavioral and white-box interpretability analyses" }, "prompting": { "status": "disclosed", "value": "research-sabotage scenarios from the Agentic Misalignment evaluation" }, "sample_selection": { "status": "disclosed", "value": "Research Sabotage evaluation set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "scenario-dependent agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_research_sabotage", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 90, PDF, section, Datasets used for evaluation, item Research Sabotage" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_research_sabotage" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_research_sandbagging", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "coding task observed by a less-capable monitor" }, "prompting": { "status": "disclosed", "value": "instructions to sabotage a task or imitate a weaker model without alerting a monitor" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_research_sandbagging", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 85, PDF, section, Research Sandbagging" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_research_sandbagging" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_reward_hack_prone_coding_tasks_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "classifier-based and hidden-test variants" }, "prompting": { "status": "disclosed", "value": "standard coding-task instructions" }, "sample_selection": { "status": "disclosed", "value": "expanded hand-selected coding tasks from the training distribution" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_reward_hack_prone_coding_tasks_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 49, PDF, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_reward_hack_prone_coding_tasks_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_seqqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench multiple-choice evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "SeqQA subtask" }, "shot_configuration": { "status": "disclosed", "value": "ten-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 133, PDF, section, SeqQA subtask and Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_seqqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_shade_arena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "long-horizon agent scaffold with a monitor that could inspect reasoning in the primary setting" }, "prompting": { "status": "disclosed", "value": "explicit main-task and harmful-side-task instructions" }, "sample_selection": { "status": "disclosed", "value": "paired main-task and side-task scenarios" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic task tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 86, PDF, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_shade_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_short_horizon_computational_biology_tasks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "containerized multi-step computational biology environment with continuous grading" }, "prompting": { "status": "disclosed", "value": "task-specific computational biology objectives" }, "sample_selection": { "status": "disclosed", "value": "pathogen design and pathogen variant-prediction tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bioinformatics analysis, variant calling, effect prediction, protein-folding prediction, and database tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_short_horizon_computational_biology_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 136, PDF, section, Short-horizon computational biology tasks" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_short_horizon_computational_biology_tasks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "answers classified as correct, incorrect, or unsure by an Anthropic grading model" }, "prompting": { "status": "disclosed", "value": "direct factual questions" }, "sample_selection": { "status": "disclosed", "value": "Simple-QA question set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no web search or external tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_simpleqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 22, PDF, section, Obscure Questions, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_subversion_strategy", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "toy deployment-detection and selective-action task" }, "prompting": { "status": "disclosed", "value": "explicit instructions to distinguish testing from deployment and act selectively" }, "sample_selection": { "status": "disclosed", "value": "varied simulated deployment sizes" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_subversion_strategy", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 84, PDF, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_subversion_strategy" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "simple scaffold without test-time compute for the primary result; a separate high-compute configuration used parallel attempts, visible-test filtering, and internal candidate selection" }, "prompting": { "status": "disclosed", "value": "minor addition encouraged extensive tool use and writing tests before implementation" }, "sample_selection": { "status": "disclosed", "value": "full SWE-bench Verified problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash and string-replacement file-editing tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "heading", "value": "Frontier intelligence, SWE-bench Verified result; Methodology, SWE-bench Verified" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "solutions graded by repository unit tests" }, "prompting": { "status": "disclosed", "value": "original issue description and relevant repository context" }, "sample_selection": { "status": "disclosed", "value": "hard subset selected by estimated human engineering time" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-engineering agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 140, PDF, sections -, Table .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_swe_bench_verified_hard_subset" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_sycophancy_eval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "behavioral and white-box interpretability analyses" }, "prompting": { "status": "disclosed", "value": "handwritten sycophancy prompts" }, "sample_selection": { "status": "disclosed", "value": "smaller handwritten sycophancy set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "none disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_sycophancy_eval", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 90, PDF, section, Datasets used for evaluation, item Sycophancy Eval" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_sycophancy_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_synthetic_honeypots", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "behavioral and white-box interpretability analyses across model snapshots" }, "prompting": { "status": "disclosed", "value": "synthetically generated agentic scenarios designed to elicit misaligned actions" }, "sample_selection": { "status": "disclosed", "value": "scenarios drawn from a distribution similar to training evaluations" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "scenario-dependent agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_synthetic_honeypots", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 90, PDF, section, Datasets used for evaluation, item Synthetic Honeypots" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_synthetic_honeypots" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "default Terminus 2 agent framework with XML parser and repeated runs on different days" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "terminal interaction" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "heading", "value": "Methodology, Terminal-Bench" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_text_based_reinforcement_learning_task", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal text-based reinforcement-learning environment" }, "prompting": { "status": "disclosed", "value": "reinforcement-learning objective" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding and machine-learning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_text_based_reinforcement_learning_task", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 143, PDF, section, Text-based reinforcement learning task" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_text_based_reinforcement_learning_task" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_time_series_forecasting", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal forecasting environment with variants of increasing difficulty" }, "prompting": { "status": "disclosed", "value": "forecasting objective" }, "sample_selection": { "status": "disclosed", "value": "disclosed task variants" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding and machine-learning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_time_series_forecasting", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 142, PDF, section, Time series forecasting" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_time_series_forecasting" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_tool_use_prompt_injection_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "technical and business tasks with embedded prompt injections" }, "prompting": { "status": "disclosed", "value": "adversarial instructions appeared in files, scripts, or command output" }, "sample_selection": { "status": "disclosed", "value": "synthetic prompt-injection cases spanning varied task types" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash commands" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_tool_use_prompt_injection_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 31, PDF, section, Tool use evaluation table" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_tool_use_prompt_injection_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vals_ai_finance_agent", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vals AI public leaderboard configuration with extended and interleaved thinking" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_vals_ai_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "heading", "value": "Methodology, Finance Agent" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_vals_ai_finance_agent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vmqa_vct", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "all correct options and no incorrect options required for an item to count as correct" }, "prompting": { "status": "disclosed", "value": "multiple-select virology questions containing text and images" }, "sample_selection": { "status": "disclosed", "value": "SecureBio multimodal virology question set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "image input" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_vmqa_vct", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 132, PDF, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_vmqa_vct" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_record_100q_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "answers classified as correct, incorrect, or unsure by an Anthropic grading model" }, "prompting": { "status": "disclosed", "value": "direct factual questions" }, "sample_selection": { "status": "disclosed", "value": "internal human-written niche factual questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no web search or external tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_x_100q_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 22, PDF, section, Obscure Questions, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_x_100q_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_revision_id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_record_2_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Airline and Telecom Agent Policy trajectories with a modified Telecom User prompt" }, "prompting": { "status": "disclosed", "value": "prompt addenda targeted known model and simulated-user failure modes" }, "sample_selection": { "status": "disclosed", "value": "Airline and Telecom tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "extended thinking with tool use" } }, "id": "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_x_2_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "heading", "value": "Methodology, τ2-bench" }, "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20250929_introducing_claude_sonnet_4_5_benchmark_x_2_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_agent_red_teaming", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external Gray Swan agent red-teaming framework on a near-final snapshot" }, "prompting": { "status": "disclosed", "value": "adversarial scenarios involving data leakage, safety overrides, malicious activity, and unauthorized tool use" }, "sample_selection": { "status": "disclosed", "value": "same ART attack scenarios used for the preceding Anthropic release" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_agent_red_teaming", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 17, PDF, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_agent_red_teaming" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_agentic_misalignment_suite", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "expanded synthetic misalignment suite; full quantitative results were withheld because of evaluation awareness" }, "prompting": { "status": "disclosed", "value": "hand-built artificial agentic scenarios" }, "sample_selection": { "status": "disclosed", "value": "scenario families involving blackmail, murder, and financial-crime framing" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "scenario-dependent agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_agentic_misalignment_suite", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 27, PDF, section, Agentic misalignment suite" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_agentic_misalignment_suite" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aime", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "default sampling with extended thinking" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "independent sampled trials" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_aime", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "heading", "value": "Benchmarks, benchmark figure row; Methodology, AIME" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_aime" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "default non-extended-thinking mode on a near-final model snapshot" }, "prompting": { "status": "disclosed", "value": "standard benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "ambiguous and disambiguated question sets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_bias_benchmark_for_question_answering", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 13, PDF, section, Tables .A and .B" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_bias_benchmark_for_question_answering" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cloningscenarios", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench assessment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Cloning Scenarios subtask" }, "shot_configuration": { "status": "disclosed", "value": "ten-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_cloningscenarios", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 37, PDF, section, LAB-Bench subset bullet, Cloning Scenarios task" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_cloningscenarios" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "capture-the-flag agent evaluation" }, "prompting": { "status": "disclosed", "value": "challenge instructions" }, "sample_selection": { "status": "disclosed", "value": "disclosed subset of Cybench challenges" }, "shot_configuration": { "status": "disclosed", "value": "multiple attempts per challenge" }, "tool_use": { "status": "disclosed", "value": "cybersecurity challenge tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 39, PDF, section, Cyber evaluations" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_cybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_figqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench assessment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "FigQA subtask" }, "shot_configuration": { "status": "disclosed", "value": "ten-shot" }, "tool_use": { "status": "disclosed", "value": "image input" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 37, PDF, section, LAB-Bench subset bullet, FigQA task" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_figqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_impossible_tasks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "agentic coding environment" }, "prompting": { "status": "disclosed", "value": "minimal-instruction and explicit anti-hardcoding variants" }, "sample_selection": { "status": "disclosed", "value": "problems intentionally designed to be impossible" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_impossible_tasks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 34, PDF, section 5, Table 5.B and Impossible Tasks description" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_impossible_tasks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_internal_prompt_injection_evaluations", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "surface-specific agent tasks, with computer-use runs also evaluated with additional safeguards" }, "prompting": { "status": "disclosed", "value": "adversarial instructions embedded in files, websites, collaboration content, or tool output" }, "sample_selection": { "status": "disclosed", "value": "internal MCP, computer-use, and tool-use test cases" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "MCP collaboration tools, computer actions, and general command tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_internal_prompt_injection_evaluations", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 18, PDF, section, Internal Prompt Injection Evaluations" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_internal_prompt_injection_evaluations" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench biological-research assessment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "ProtocolQA, SeqQA, Cloning Scenarios, and FigQA" }, "shot_configuration": { "status": "disclosed", "value": "ten-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_lab_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 37, PDF, section, LAB-Bench subset bullet" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_lab_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_long_form_virology_task_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "long-form task with sequence-design and protocol-design components" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Long-Form Virology Task 1" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic task tools not further disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_long_form_virology_task_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 37, PDF, section, Long-Form Virology Task 1 bullet and discussion" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_long_form_virology_task_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_long_form_virology_task_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "long-form task with sequence-design and protocol-design components" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Long-Form Virology Task 2" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic task tools not further disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_long_form_virology_task_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 37, PDF, section, Long-Form Virology Task 2 bullet and discussion" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_long_form_virology_task_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "default sampling with extended thinking" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "non-English language subsets" }, "shot_configuration": { "status": "disclosed", "value": "repeated runs" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "heading", "value": "Benchmarks, benchmark figure row; Methodology, MMMLU" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official OSWorld-Verified framework with a bounded-step agent and per-step thinking" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "OSWorld tasks" }, "shot_configuration": { "status": "disclosed", "value": "repeated runs" }, "tool_use": { "status": "disclosed", "value": "computer interaction" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "heading", "value": "Benchmarks, benchmark figure row; Methodology, OSWorld" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_osworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench assessment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "ProtocolQA subtask" }, "shot_configuration": { "status": "disclosed", "value": "ten-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_protocolqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 37, PDF, section, LAB-Bench subset bullet, ProtocolQA task" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_protocolqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_reward_hack_prone_coding_tasks_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "classifier-based and hidden-test variants" }, "prompting": { "status": "disclosed", "value": "standard coding-task instructions" }, "sample_selection": { "status": "disclosed", "value": "hand-selected coding tasks from the training distribution" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_reward_hack_prone_coding_tasks_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 34, PDF, section 5, Table 5.B" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_reward_hack_prone_coding_tasks_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_seqqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench assessment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "SeqQA subtask" }, "shot_configuration": { "status": "disclosed", "value": "ten-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_seqqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 37, PDF, section, LAB-Bench subset bullet, SeqQA task" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_seqqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_shade_arena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "agent scaffold with a monitor that could inspect reasoning" }, "prompting": { "status": "disclosed", "value": "main-task and harmful-side-task instructions" }, "sample_selection": { "status": "disclosed", "value": "paired main-task and side-task scenarios" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic task tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 28, PDF, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_shade_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "simple scaffold without test-time compute and with default sampling" }, "prompting": { "status": "disclosed", "value": "minor addition encouraged extensive tool use and writing tests before implementation" }, "sample_selection": { "status": "disclosed", "value": "full SWE-bench Verified problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash and string-replacement file-editing tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "heading", "value": "Benchmarks, benchmark figure row; Methodology, SWE-bench Verified" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified_hard_subset", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "autonomy checkpoint evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "hard SWE-bench Verified subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-engineering agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_swe_bench_verified_hard_subset", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 38, PDF, section, Autonomy evaluations" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_swe_bench_verified_hard_subset" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_synthesis_screening_evasion", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "criterion-based synthesis-screening-evasion evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 38, PDF, section, Synthesis Screening Evasion discussion" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_synthesis_screening_evasion" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "default Terminus 2 agent framework with XML parser, using thinking and non-thinking runs" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "single-attempt runs" }, "tool_use": { "status": "disclosed", "value": "terminal interaction" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_terminal_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "heading", "value": "Benchmarks, benchmark figure row; Methodology, Terminal-Bench" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vmqa_vct", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multimodal virology question evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "VMQA/VCT question set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "image input" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_vmqa_vct", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "page", "value": "page 37, PDF, section, VMQA/VCT bullet" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_vmqa_vct" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_revision_id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_record_2_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Airline and Telecom Agent Policy trajectories with default sampling" }, "prompting": { "status": "disclosed", "value": "prompt addenda targeted known model and simulated-user failure modes" }, "sample_selection": { "status": "disclosed", "value": "Airline and Telecom tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "extended thinking with tool use" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_x_2_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "locator": { "kind": "heading", "value": "Benchmarks, benchmark figure row; Methodology, τ2-bench" }, "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251015_introducing_claude_haiku_4_5_benchmark_x_2_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "release_id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20251015_introducing_claude_haiku_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aider_polyglot", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aider_polyglot", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Evaluating Claude Opus, benchmark highlight row Aider Polyglot" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aider_polyglot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20251124_introducing_claude_opus_4_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20251124_introducing_claude_opus_4_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "competition questions" }, "sample_selection": { "status": "disclosed", "value": "AIME 2025 questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and Python-tool configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aime_2025", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 32, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC Prize Foundation evaluation" }, "prompting": { "status": "disclosed", "value": "input-output grid demonstrations" }, "sample_selection": { "status": "disclosed", "value": "private validation set" }, "shot_configuration": { "status": "disclosed", "value": "few-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 28, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_arc_agi_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC Prize Foundation evaluation" }, "prompting": { "status": "disclosed", "value": "input-output grid demonstrations" }, "sample_selection": { "status": "disclosed", "value": "private validation set" }, "shot_configuration": { "status": "disclosed", "value": "few-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 29, section, Figure .B" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_browsecomp_plus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "benchmark questions with a separately disclosed grading prompt" }, "sample_selection": { "status": "disclosed", "value": "fixed human-verified document corpus" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "embedding search; configurations also tested document fetch, memory, and result clearing" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_browsecomp_plus", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 21, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_browsecomp_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic implementation of a Cybench subset" }, "prompting": { "status": "disclosed", "value": "challenge instructions" }, "sample_selection": { "status": "disclosed", "value": "subset selected subject to infrastructure constraints" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "cybersecurity challenge environment" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 150, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberGym suite" }, "prompting": { "status": "disclosed", "value": "high-level vulnerability descriptions" }, "sample_selection": { "status": "disclosed", "value": "published vulnerability tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools plus an interleaved think tool" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 30, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_finance_agent", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vals AI evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "FinanceAgent test set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 30, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_finance_agent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "multiple-choice questions" }, "sample_selection": { "status": "disclosed", "value": "Diamond subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 32, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after contamination screening" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and web-search, web-fetch, and code-execution configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_humanity_s_last_exam", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 31, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_figqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench FigQA evaluation" }, "prompting": { "status": "disclosed", "value": "zero-shot" }, "sample_selection": { "status": "disclosed", "value": "FigQA subset" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and image-cropping configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 33, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_lab_bench_figqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MCP-Atlas server environments" }, "prompting": { "status": "disclosed", "value": "benchmark workflow requests" }, "sample_selection": { "status": "disclosed", "value": "benchmark workflow tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "authentic Model Context Protocol tools and APIs" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 30, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "all reported non-English language pairings" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 33, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "validation split" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mmmu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 33, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OSWorld default environment" }, "prompting": { "status": "disclosed", "value": "default benchmark prompts" }, "sample_selection": { "status": "disclosed", "value": "default task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "mouse and keyboard computer-use actions" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 27, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_osworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_spreadsheetbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "custom Anthropic spreadsheet harness" }, "prompting": { "status": "disclosed", "value": "spreadsheet task instructions" }, "sample_selection": { "status": "disclosed", "value": "full published problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash, text editing, Python, spreadsheet, and data-analysis libraries" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_spreadsheetbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 31, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_spreadsheetbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench Multilingual evaluation" }, "prompting": { "status": "disclosed", "value": "standard benchmark prompting" }, "sample_selection": { "status": "disclosed", "value": "multilingual programming-language problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 20, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench Pro evaluation" }, "prompting": { "status": "disclosed", "value": "standard benchmark prompting" }, "sample_selection": { "status": "disclosed", "value": "published problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 20, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench evaluation" }, "prompting": { "status": "disclosed", "value": "standard benchmark prompting" }, "sample_selection": { "status": "disclosed", "value": "human-verified solvable subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 20, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Terminus-2 in the Harbor scaffold" }, "prompting": { "status": "disclosed", "value": "benchmark task prompts" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "terminal and command-line tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 20, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vending_bench_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vending-Bench context-management system" }, "prompting": { "status": "disclosed", "value": "simulated business-management task instructions" }, "sample_selection": { "status": "disclosed", "value": "simulated business run" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "simulated email, supplier, inventory, and pricing operations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 29, section" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_vending_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_webarena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single policy model with official WebArena grader" }, "prompting": { "status": "disclosed", "value": "general-purpose system prompts without website-specific prompt engineering" }, "sample_selection": { "status": "disclosed", "value": "WebArena task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Computer Use API with screenshot and DOM browser tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_webarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 34, section, Tables .A and .B" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_webarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_record_2_bench_airline_corrected", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "vanilla prompt with a failure-mode addendum" }, "sample_selection": { "status": "disclosed", "value": "corrected Airline section" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "programmatic airline APIs" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_x_2_bench_airline_corrected", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 26, section, Table .A, Airline (corrected) row" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_x_2_bench_airline_corrected" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_record_2_bench_airline_original", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "vanilla prompt with a failure-mode addendum" }, "sample_selection": { "status": "disclosed", "value": "original Airline section" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "programmatic airline APIs" } }, "id": "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_x_2_bench_airline_original", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 26, section, Table .A, Airline (original) row" }, "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20251124_introducing_claude_opus_4_5_benchmark_x_2_bench_airline_original" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "release_id": "release_anthropic_20251124_introducing_claude_opus_4_5", "review_status": "verified", "source_id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_revision_id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "competition questions" }, "sample_selection": { "status": "disclosed", "value": "AIME 2025 questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_aime_2025", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 25, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC Prize Foundation evaluation" }, "prompting": { "status": "disclosed", "value": "input-output grid demonstrations" }, "sample_selection": { "status": "disclosed", "value": "private validation set" }, "shot_configuration": { "status": "disclosed", "value": "few-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_arc_agi_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC Prize Foundation evaluation" }, "prompting": { "status": "disclosed", "value": "input-output grid demonstrations" }, "sample_selection": { "status": "disclosed", "value": "private validation set" }, "shot_configuration": { "status": "disclosed", "value": "few-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biglaw_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Harvey evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biglaw_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "First impressions, Harvey quotation" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biglaw_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260205_introducing_claude_opus_4_6", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260205_introducing_claude_opus_4_6", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biomysterybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal Anthropic life-science suite" }, "prompting": { "status": "disclosed", "value": "questions over unprocessed biological datasets" }, "sample_selection": { "status": "disclosed", "value": "internally developed analytical biology tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash, code execution, and package installation" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 45, section, Computational Biology, BioMysteryBench" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biomysterybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biopipelinebench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal Anthropic life-science suite" }, "prompting": { "status": "disclosed", "value": "workflow task instructions" }, "sample_selection": { "status": "disclosed", "value": "internally developed bioinformatics workflows" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash, code execution, and package installation" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biopipelinebench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 45, section, Computational Biology, BioPipelineBench" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_biopipelinebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-agent search harness with context compaction" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "BrowseComp questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, and programmatic tool calling" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 39, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CharXiv Reasoning evaluation" }, "prompting": { "status": "disclosed", "value": "chart questions" }, "sample_selection": { "status": "disclosed", "value": "validation-split questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and image-cropping configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 36, section, Figure .C" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic implementation of a Cybench subset" }, "prompting": { "status": "disclosed", "value": "challenge instructions" }, "sample_selection": { "status": "disclosed", "value": "subset selected subject to infrastructure constraints" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "cybersecurity challenge environment" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 202, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberGym suite" }, "prompting": { "status": "disclosed", "value": "high-level vulnerability descriptions" }, "sample_selection": { "status": "disclosed", "value": "targeted vulnerability-reproduction tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools plus an interleaved think tool" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 29, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_deepsearchqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-agent search harness with context compaction" }, "prompting": { "status": "disclosed", "value": "multi-step information-seeking prompts" }, "sample_selection": { "status": "disclosed", "value": "DeepSearchQA prompt set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, and programmatic tool calling" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 42, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_deepsearchqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_finance_agent", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vals AI evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Finance Agent benchmark tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 26, section, Table under Finance Agent" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_finance_agent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gdpval_aa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "independent Artificial Analysis evaluation with blind pairwise grading" }, "prompting": { "status": "disclosed", "value": "professional task instructions" }, "sample_selection": { "status": "disclosed", "value": "tasks from the GDPval gold dataset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell access and web browsing in an agentic loop" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 24, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_gdpval_aa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "multiple-choice questions" }, "sample_selection": { "status": "disclosed", "value": "Diamond subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 24, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_graphwalks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic implementation with disclosed scoring and label corrections" }, "prompting": { "status": "disclosed", "value": "clarified exact-depth BFS prompt and parent-node prompts" }, "sample_selection": { "status": "disclosed", "value": "BFS and parent-node variants across long-context subsets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no external tools disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 30, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_graphwalks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after contamination screening" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and web-search, web-fetch, programmatic-calling, and code-execution configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_humanity_s_last_exam", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 41, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_figqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench FigQA evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "FigQA subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and image-cropping configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 34, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_lab_bench_figqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MCP-Atlas server environments" }, "prompting": { "status": "disclosed", "value": "benchmark workflow requests" }, "sample_selection": { "status": "disclosed", "value": "benchmark workflow tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "authentic Model Context Protocol tools and APIs" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 29, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "all reported non-English language pairings" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 25, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MMMU-Pro evaluation" }, "prompting": { "status": "disclosed", "value": "standard and vision-format questions" }, "sample_selection": { "status": "disclosed", "value": "standard and vision formats" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and image-cropping configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmu_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 35, section, Figure .B" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_openai_mrcr_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "published OpenAI dataset with the v2 fix" }, "prompting": { "status": "disclosed", "value": "repeated-request retrieval prompts" }, "sample_selection": { "status": "disclosed", "value": "hardest needle variants across long-context bins" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no external tools disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_openai_mrcr_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 30, section, Table .A and Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_openai_mrcr_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_openrca", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "benchmark author’s agent harness" }, "prompting": { "status": "disclosed", "value": "failure-case root-cause requests" }, "sample_selection": { "status": "disclosed", "value": "published enterprise-system failure cases" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "telemetry inspection through the benchmark agent" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_openrca", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 20, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_openrca" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OSWorld-Verified default settings" }, "prompting": { "status": "disclosed", "value": "default benchmark prompts" }, "sample_selection": { "status": "disclosed", "value": "verified task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "mouse and keyboard actions in a live Ubuntu virtual machine" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_real_world_finance", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic agentic harness with rubric and preference grading" }, "prompting": { "status": "disclosed", "value": "real analyst workflow tasks" }, "sample_selection": { "status": "disclosed", "value": "difficult tasks drawn from several finance verticals" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "code execution and task-dependent productivity tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_real_world_finance", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 27, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_real_world_finance" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench Multilingual evaluation" }, "prompting": { "status": "disclosed", "value": "standard benchmark prompting" }, "sample_selection": { "status": "disclosed", "value": "multilingual programming-language problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 19, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench evaluation" }, "prompting": { "status": "disclosed", "value": "standard prompting; a separate modified prompt was also investigated" }, "sample_selection": { "status": "disclosed", "value": "human-verified solvable subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 19, Table .A and section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Terminus-2 in the Harbor scaffold" }, "prompting": { "status": "disclosed", "value": "benchmark task prompts with default parser" }, "sample_selection": { "status": "disclosed", "value": "full task set across staggered batches" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "terminal and command-line tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 19, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vending_bench_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vending-Bench context-management system" }, "prompting": { "status": "disclosed", "value": "simulated business-management task instructions" }, "sample_selection": { "status": "disclosed", "value": "simulated business run" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "simulated email, supplier, inventory, and pricing operations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 29, section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_vending_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_webarena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single policy model with modified official grader" }, "prompting": { "status": "disclosed", "value": "general-purpose system prompts" }, "sample_selection": { "status": "disclosed", "value": "WebArena task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Computer Use API with screenshot and DOM browser tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_webarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 37, section, Tables .A and .B" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_webarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_record_2_bench_retail", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "τ2-bench simulation" }, "prompting": { "status": "disclosed", "value": "domain-policy interaction prompts" }, "sample_selection": { "status": "disclosed", "value": "Retail section" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "programmatic retail APIs" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_x_2_bench_retail", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 18, Table .A Retail row and section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_x_2_bench_retail" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_record_2_bench_telecom", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "τ2-bench simulation" }, "prompting": { "status": "disclosed", "value": "domain-policy interaction prompts" }, "sample_selection": { "status": "disclosed", "value": "Telecom section" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "programmatic telecom APIs" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_x_2_bench_telecom", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 18, Table .A Telecom row and section" }, "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260205_introducing_claude_opus_4_6_benchmark_x_2_bench_telecom" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_anthropic_20260205_introducing_claude_opus_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_revision_id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "competition questions" }, "sample_selection": { "status": "disclosed", "value": "AIME 2025 questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_aime_2025", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 23, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC Prize Foundation evaluation" }, "prompting": { "status": "disclosed", "value": "input-output grid demonstrations" }, "sample_selection": { "status": "disclosed", "value": "private validation set" }, "shot_configuration": { "status": "disclosed", "value": "few-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 20, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_arc_agi_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC Prize Foundation evaluation" }, "prompting": { "status": "disclosed", "value": "input-output grid demonstrations" }, "sample_selection": { "status": "disclosed", "value": "private validation set" }, "shot_configuration": { "status": "disclosed", "value": "few-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 20, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biomysterybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal Anthropic life-science suite" }, "prompting": { "status": "disclosed", "value": "questions over unprocessed biological datasets" }, "sample_selection": { "status": "disclosed", "value": "internally developed analytical biology tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash, code execution, and package installation" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 49, section, Computational Biology, BioMysteryBench" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_biomysterybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biopipelinebench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal Anthropic life-science suite" }, "prompting": { "status": "disclosed", "value": "workflow task instructions" }, "sample_selection": { "status": "disclosed", "value": "internally developed bioinformatics workflows" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bash, code execution, and package installation" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_biopipelinebench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 49, section, Computational Biology, BioPipelineBench" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_biopipelinebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-agent search harness with context compaction" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "BrowseComp questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, and programmatic tool calling" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 44, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CharXiv Reasoning evaluation" }, "prompting": { "status": "disclosed", "value": "chart questions" }, "sample_selection": { "status": "disclosed", "value": "validation-split questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and image-cropping configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 36, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic implementation of a Cybench subset" }, "prompting": { "status": "disclosed", "value": "challenge instructions" }, "sample_selection": { "status": "disclosed", "value": "subset selected subject to infrastructure constraints" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "cybersecurity challenge environment" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 125, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberGym suite" }, "prompting": { "status": "disclosed", "value": "high-level vulnerability descriptions" }, "sample_selection": { "status": "disclosed", "value": "targeted vulnerability-reproduction tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools plus an interleaved think tool" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 28, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_deepsearchqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "search harness with context compaction; a multi-agent variant was also reported" }, "prompting": { "status": "disclosed", "value": "multi-step information-seeking prompts" }, "sample_selection": { "status": "disclosed", "value": "DeepSearchQA prompt set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, and programmatic tool calling" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 47, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_deepsearchqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_finance_agent", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vals AI evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Finance Agent benchmark tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 24, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_finance_agent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_financial_services_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Hebbia evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_financial_services_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "heading", "value": "Evaluating Claude Sonnet, Hebbia quotation" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_financial_services_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260217_introducing_claude_sonnet_4_6", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260217_introducing_claude_sonnet_4_6", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gdpval_aa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "independent Artificial Analysis evaluation with blind pairwise grading" }, "prompting": { "status": "disclosed", "value": "professional task instructions" }, "sample_selection": { "status": "disclosed", "value": "tasks from the GDPval gold dataset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell access and web browsing in an agentic loop" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 22, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gdpval_aa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_global_mmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "GMMLU evaluation" }, "prompting": { "status": "disclosed", "value": "provider-default prompting with structured JSON output" }, "sample_selection": { "status": "disclosed", "value": "languages grouped by resource level" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_global_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 41, section, GMMLU results table" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_global_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "multiple-choice questions" }, "sample_selection": { "status": "disclosed", "value": "Diamond subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 22, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_graphwalks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic implementation with disclosed corrections" }, "prompting": { "status": "disclosed", "value": "clarified exact-depth BFS prompt and parent-node prompts" }, "sample_selection": { "status": "disclosed", "value": "BFS and parent-node variants across long-context subsets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no external tools disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 32, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_graphwalks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after contamination screening" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and web-search, web-fetch, programmatic-calling, and code-execution configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_humanity_s_last_exam", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 46, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_figqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench FigQA evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "FigQA subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and image-cropping configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 34, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_lab_bench_figqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MCP-Atlas server environments" }, "prompting": { "status": "disclosed", "value": "benchmark workflow requests" }, "sample_selection": { "status": "disclosed", "value": "benchmark workflow tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "authentic Model Context Protocol tools and APIs" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 28, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_medcalc_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "multi-turn code-augmented MedCalc-Bench methodology" }, "prompting": { "status": "disclosed", "value": "instructions to write and execute code for clinical calculations" }, "sample_selection": { "status": "disclosed", "value": "latest verified dataset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Python REPL" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_medcalc_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 51, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_medcalc_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_milu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MILU evaluation" }, "prompting": { "status": "disclosed", "value": "provider-default prompting with structured JSON output" }, "sample_selection": { "status": "disclosed", "value": "reported Indic languages and English" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_milu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 41, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_milu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "all reported non-English language pairings" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 23, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MMMU-Pro evaluation with separate Claude Sonnet 4 grading" }, "prompting": { "status": "disclosed", "value": "updated prompt format without the prior step-by-step prefix" }, "sample_selection": { "status": "disclosed", "value": "standard and vision formats" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and image-cropping configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mmmu_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 35, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_officeqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Databricks evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "heading", "value": "Evaluating Claude Sonnet, Databricks quotation" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_officeqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260217_introducing_claude_sonnet_4_6", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260217_introducing_claude_sonnet_4_6", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_openai_mrcr_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "published OpenAI dataset with the v2 fix" }, "prompting": { "status": "disclosed", "value": "repeated-request retrieval prompts" }, "sample_selection": { "status": "disclosed", "value": "hardest needle variants across long-context bins" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no external tools disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_openai_mrcr_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 29, section, Table .A and Figure .B" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_openai_mrcr_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_openrca", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "benchmark author’s agent harness" }, "prompting": { "status": "disclosed", "value": "failure-case root-cause requests" }, "sample_selection": { "status": "disclosed", "value": "published enterprise-system failure cases" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "telemetry inspection through the benchmark agent" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_openrca", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 17, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_openrca" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OSWorld-Verified default settings" }, "prompting": { "status": "disclosed", "value": "default benchmark prompts" }, "sample_selection": { "status": "disclosed", "value": "verified task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "mouse and keyboard actions in a live Ubuntu virtual machine" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 19, section, Figures .A and .B" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_real_world_finance", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic agentic harness with rubric and preference grading" }, "prompting": { "status": "disclosed", "value": "real analyst workflow tasks" }, "sample_selection": { "status": "disclosed", "value": "difficult tasks drawn from several finance verticals" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "code execution and task-dependent productivity tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_real_world_finance", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 25, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_real_world_finance" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench Multilingual evaluation" }, "prompting": { "status": "disclosed", "value": "standard benchmark prompting" }, "sample_selection": { "status": "disclosed", "value": "multilingual programming-language problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 16, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench evaluation" }, "prompting": { "status": "disclosed", "value": "standard prompting; a separate modified prompt was also investigated" }, "sample_selection": { "status": "disclosed", "value": "human-verified solvable subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 15, Table .A and section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Terminus-2 in the Harbor scaffold" }, "prompting": { "status": "disclosed", "value": "benchmark task prompts with default parser" }, "sample_selection": { "status": "disclosed", "value": "full task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "terminal and command-line tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 16, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vending_bench_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vending-Bench context-management system" }, "prompting": { "status": "disclosed", "value": "simulated business-management task instructions" }, "sample_selection": { "status": "disclosed", "value": "simulated business run" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "simulated email, supplier, inventory, and pricing operations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 27, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vending_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vending_bench_arena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vending-Bench Arena" }, "prompting": { "status": "disclosed", "value": "simulated business-management instructions" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "simulated business operations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vending_bench_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "heading", "value": "Evaluating Claude Sonnet, Vending-Bench Arena paragraphs and chart caption" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_vending_bench_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260217_introducing_claude_sonnet_4_6", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260217_introducing_claude_sonnet_4_6", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_webarena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single policy model with modified official grader" }, "prompting": { "status": "disclosed", "value": "general-purpose system prompts" }, "sample_selection": { "status": "disclosed", "value": "WebArena task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Computer Use API with screenshot and DOM browser tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_webarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 37, section, Table .A" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_webarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_webarena_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "official prompts" }, "sample_selection": { "status": "disclosed", "value": "full verified set and hard subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Computer Use API with screenshot and DOM browser tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_webarena_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 38, section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_webarena_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_record_2_bench_retail", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "τ2-bench simulation" }, "prompting": { "status": "disclosed", "value": "domain-policy interaction prompts" }, "sample_selection": { "status": "disclosed", "value": "Retail section" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "programmatic retail APIs" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_x_2_bench_retail", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 15, Table .A Retail row and section" }, "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260217_introducing_claude_sonnet_4_6_benchmark_x_2_bench_retail" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "release_id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "review_status": "verified", "source_id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_revision_id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC Prize Foundation evaluation" }, "prompting": { "status": "disclosed", "value": "input-output grid demonstrations" }, "sample_selection": { "status": "disclosed", "value": "private validation set" }, "shot_configuration": { "status": "disclosed", "value": "few-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 212, section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_arc_agi_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC Prize Foundation evaluation" }, "prompting": { "status": "disclosed", "value": "input-output grid demonstrations" }, "sample_selection": { "status": "disclosed", "value": "private validation set" }, "shot_configuration": { "status": "disclosed", "value": "few-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 212, section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biglaw_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Harvey evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_biglaw_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Testing Claude Opus, Harvey quotation" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_biglaw_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260416_introducing_claude_opus_4_7", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260416_introducing_claude_opus_4_7", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-agent search harness with context compaction" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "BrowseComp questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, programmatic tool calling, and code execution" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 198, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CharXiv Reasoning evaluation" }, "prompting": { "status": "disclosed", "value": "chart questions" }, "sample_selection": { "status": "disclosed", "value": "validation-split questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and Python-tool configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 204, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cursorbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Cursor evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cursorbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Testing Claude Opus, Cursor quotation" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cursorbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260416_introducing_claude_opus_4_7", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260416_introducing_claude_opus_4_7", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic implementation of a Cybench subset" }, "prompting": { "status": "disclosed", "value": "challenge instructions" }, "sample_selection": { "status": "disclosed", "value": "subset selected subject to infrastructure constraints" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "cybersecurity challenge environment plus an interleaved think tool" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 49, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberGym suite with updated harness parameters" }, "prompting": { "status": "disclosed", "value": "high-level vulnerability descriptions" }, "sample_selection": { "status": "disclosed", "value": "targeted vulnerability-reproduction tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding tools plus an interleaved think tool" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 50, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_deepsearchqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "search harness with context compaction" }, "prompting": { "status": "disclosed", "value": "multi-step information-seeking prompts" }, "sample_selection": { "status": "disclosed", "value": "DeepSearchQA prompt set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, and programmatic tool calling" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 199, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_deepsearchqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_draco", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "task prompts with a final-report result-span instruction" }, "sample_selection": { "status": "disclosed", "value": "curated tasks derived from real user queries" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, programmatic tool calling, and code execution" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_draco", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 201, section, Figures .A and .B" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_draco" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_finance_agent", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vals AI evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Finance Agent benchmark tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 210, section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_finance_agent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_firefox_147_exploitation_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Firefox content-process simulation without browser sandbox mitigations" }, "prompting": { "status": "disclosed", "value": "instructions naming a crash category as the exploration starting point" }, "sample_selection": { "status": "disclosed", "value": "crash categories discovered during prior vulnerability research" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "containerized JavaScript-engine shell and testing harness" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_firefox_147_exploitation_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 51, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_firefox_147_exploitation_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gdpval_aa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "independent Artificial Analysis evaluation with blind pairwise grading" }, "prompting": { "status": "disclosed", "value": "professional task instructions" }, "sample_selection": { "status": "disclosed", "value": "tasks from the GDPval gold database" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell access and web browsing in an agentic loop" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 211, section, Figure .A" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_gdpval_aa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "multiple-choice questions" }, "sample_selection": { "status": "disclosed", "value": "Diamond subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 193, section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_graphwalks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic implementation with disclosed scoring correction" }, "prompting": { "status": "disclosed", "value": "clarified exact-depth BFS prompt and parent-node prompts" }, "sample_selection": { "status": "disclosed", "value": "BFS and parent-node long-context variants" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no external tools disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 194, section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_graphwalks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after contamination screening" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and web-search, web-fetch, programmatic-calling, and code-execution configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_humanity_s_last_exam", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 196, section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_figqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench FigQA evaluation at the model’s supported image resolution" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "FigQA subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and Python-tool configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 203, section, Figures .A and .B" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_lab_bench_figqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "refreshed Scale AI MCP-Atlas harness" }, "prompting": { "status": "disclosed", "value": "benchmark workflow requests" }, "sample_selection": { "status": "disclosed", "value": "public leaderboard configuration; an extended configuration was also evaluated" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "authentic Model Context Protocol tools and APIs" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 210, section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anthropic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark questions" }, "sample_selection": { "status": "disclosed", "value": "all reported non-English language pairings" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mmmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 193, section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_officeqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OfficeQA exact-match evaluation" }, "prompting": { "status": "disclosed", "value": "office-style questions grounded in supplied files" }, "sample_selection": { "status": "disclosed", "value": "OfficeQA task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document, spreadsheet, and presentation processing" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 209, section and Table .A" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_officeqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_officeqa_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OfficeQA Pro exact-match evaluation" }, "prompting": { "status": "disclosed", "value": "office-style questions grounded in supplied files" }, "sample_selection": { "status": "disclosed", "value": "OfficeQA Pro task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document, spreadsheet, and presentation processing" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_officeqa_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 209, section and Table .A" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_officeqa_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_openai_mrcr_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "published OpenAI dataset with the v2 fix" }, "prompting": { "status": "disclosed", "value": "repeated-request retrieval prompts" }, "sample_selection": { "status": "disclosed", "value": "hardest needle variants across long-context bins" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no external tools disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_openai_mrcr_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 195, section, Figures .A and .B" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_openai_mrcr_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "updated OSWorld agent scaffolding" }, "prompting": { "status": "disclosed", "value": "default task prompts with disclosed minor scaffold refinements" }, "sample_selection": { "status": "disclosed", "value": "reported task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "mouse and keyboard actions in a live Ubuntu virtual machine" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_osworld", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 207, section, Figures .A and .B" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_osworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_rakuten_swe_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Rakuten evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "production tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_rakuten_swe_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Testing Claude Opus, Rakuten quotation" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_rakuten_swe_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260416_introducing_claude_opus_4_7", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260416_introducing_claude_opus_4_7", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ScreenSpot-Pro evaluation at the model’s supported image resolution" }, "prompting": { "status": "disclosed", "value": "natural-language element-location instructions" }, "sample_selection": { "status": "disclosed", "value": "expert-annotated professional-application tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "separate no-tool and Python-tool configurations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_screenspot_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 205, section, ScreenSpot-Pro figures" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench Multilingual evaluation" }, "prompting": { "status": "disclosed", "value": "standard benchmark prompting" }, "sample_selection": { "status": "disclosed", "value": "multilingual programming-language problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 191, Table .A and section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multimodal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal implementation of SWE-bench Multimodal" }, "prompting": { "status": "disclosed", "value": "standard benchmark prompting with visual context" }, "sample_selection": { "status": "disclosed", "value": "published multimodal problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 191, Table .A and section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_multimodal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench Pro evaluation" }, "prompting": { "status": "disclosed", "value": "standard benchmark prompting" }, "sample_selection": { "status": "disclosed", "value": "published Pro problem set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 191, Table .A and section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench evaluation" }, "prompting": { "status": "disclosed", "value": "standard benchmark prompting" }, "sample_selection": { "status": "disclosed", "value": "human-verified solvable subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "repository and coding tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 191, Table .A and section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Terminus-2 in the Harbor scaffold" }, "prompting": { "status": "disclosed", "value": "benchmark task prompts with default parser" }, "sample_selection": { "status": "disclosed", "value": "full unique task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "terminal and command-line tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_terminal_bench_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 191, Table .A and section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_usamo_2026", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MathArena-style neutral rewrite and multi-judge grading" }, "prompting": { "status": "disclosed", "value": "proof problems" }, "sample_selection": { "status": "disclosed", "value": "USAMO 2026 problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_usamo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 194, section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_usamo_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vending_bench_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vending-Bench context-management system" }, "prompting": { "status": "disclosed", "value": "simulated business-management task instructions" }, "sample_selection": { "status": "disclosed", "value": "simulated business runs" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "simulated email, supplier, inventory, and pricing operations" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 210, section" }, "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260416_introducing_claude_opus_4_7_benchmark_vending_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_anthropic_20260416_introducing_claude_opus_4_7", "review_status": "verified", "source_id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_revision_id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aav_capsid_packaging_prediction", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "biological context disclosed to the model" }, "sample_selection": { "status": "disclosed", "value": "unpublished therapeutically relevant candidates" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "reasoning-only and container conditions were evaluated" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_aav_capsid_packaging_prediction", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 26, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_aav_capsid_packaging_prediction" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arxivmath", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MathArena evaluation" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "recent self-contained paper-derived problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_arxivmath", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 199, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_arxivmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_automation_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "AutomationBench" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "tasks seeded from real customer workflow patterns" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "business-application tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_automationbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 227, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_automationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "ambiguous questions include an unknown-answer condition" }, "sample_selection": { "status": "disclosed", "value": "standard benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no tools disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_bias_benchmark_for_question_answering", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 68, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_bias_benchmark_for_question_answering" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biomysterybench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "verified evaluation" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "unprocessed biological datasets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "computational biology tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biomysterybench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 234, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biomysterybench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biopipelinebench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "verified workflow environment" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "verified bioinformatics workflows" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bioinformatics tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biopipelinebench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 233, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_biopipelinebench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_black_box_rna_sequence_modeling_and_design", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "containerized environment with standard scientific Python libraries" }, "prompting": { "status": "disclosed", "value": "minimal prompting" }, "sample_selection": { "status": "disclosed", "value": "unpublished sequences and objectives" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "containerized scientific-computing tools and GPU access" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_black_box_rna_sequence_modeling_and_design", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 22, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_black_box_rna_sequence_modeling_and_design" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-agent and multi-agent harnesses with context compaction" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at maximum effort" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after the disclosed blocklist" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, programmatic tool calling, and code execution" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 203, section; multi-agent results on" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_chartmuseum", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "expert-annotated real-world chart questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_chartmuseum", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 217, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_chartmuseum" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_chartqapro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world charts and associated questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "tool-free and tool-enabled conditions" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_chartqapro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 216, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_chartqapro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world scientific charts" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "tool-free and Python-tool conditions" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 219, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cursorbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "multiple effort levels" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cursorbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Collaborating with Opus, early-tester quotation" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cursorbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260528_introducing_claude_opus_4_8", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260528_introducing_claude_opus_4_8", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberGym" }, "prompting": { "status": "disclosed", "value": "high-level weakness description supplied" }, "sample_selection": { "status": "disclosed", "value": "real open-source projects" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic software-development tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 51, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_deep_research_accuracy_completeness_and_objectivity", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "context compaction" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "curated deep-research tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, programmatic tool calling, and code execution" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_deep_research_accuracy_completeness_and_objectivity", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 207, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_deep_research_accuracy_completeness_and_objectivity" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_deepsearchqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multi-step information-seeking prompts" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web-research tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 205, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_deepsearchqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_exploitbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "vulnerable engine environment" }, "prompting": { "status": "disclosed", "value": "vulnerable build and corresponding patch supplied" }, "sample_selection": { "status": "disclosed", "value": "patched engine vulnerabilities" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-development and execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_exploitbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 50, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_exploitbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_finance_agent", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external evaluator harness" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "public-company filing tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "financial-document research tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 224, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_finance_agent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_firefox_exploit_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "browser vulnerability environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "patched browser vulnerabilities" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-development and execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_firefox_exploit_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 52, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_firefox_exploit_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_frontierswe", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "open-ended long-horizon engineering problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_frontierswe", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 197, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_frontierswe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gdpval_aa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external evaluation framework" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world professional tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-dependent professional tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 226, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_gdpval_aa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_global_mmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_global_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 229, section; results on" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_global_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "Diamond subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 198, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_graphwalks_bfs_256k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "GraphWalks" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "BFS long-context variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks_bfs_256k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 200, section, evaluation table row" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks_bfs_256k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_graphwalks_parents_256k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "GraphWalks" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "parent-retrieval long-context variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks_parents_256k", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 200, section, evaluation table row" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_graphwalks_parents_256k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_healthbench_professional", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "rubric-based model grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "physician-authored conversations" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_healthbench_professional", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 228, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_healthbench_professional" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after the disclosed blocklist" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "both without tools and with tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_humanity_s_last_exam", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 202, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_include", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 229, section; results on" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_figqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "biology-research figures and questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 218, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_lab_bench_figqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_labbench2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LABBench2" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "biology research questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "live-web research tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_labbench2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 235, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_labbench2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_latchbio_bioinformatics", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LatchBio evaluation environment" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world bioinformatics problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bioinformatics tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_latchbio_bioinformatics", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 234, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_latchbio_bioinformatics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_legal_agent_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Legal Agent Benchmark" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "public benchmark tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document and spreadsheet tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_legal_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 224, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_legal_agent_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MCP-Atlas" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world tool-use workflows" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Model Context Protocol tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 225, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_minimal_linuxbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Minimal-LinuxBench" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Linux command-line tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_minimal_linuxbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 133, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_minimal_linuxbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mask", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "MASK dataset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_model_alignment_between_statements_and_knowledge", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 118, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_model_alignment_between_statements_and_knowledge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_multi_task_indic_language_understanding_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "Indic-language benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_multi_task_indic_language_understanding_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 229, section; results on" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_multi_task_indic_language_understanding_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_officeqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OfficeQA" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "historical treasury-document corpus" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document search and reasoning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 223, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_officeqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_online_mind2web", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "browser interaction" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_online_mind2web", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Collaborating with Opus, early-tester quotation" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_online_mind2web" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260528_introducing_claude_opus_4_8", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260528_introducing_claude_opus_4_8", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_oss_fuzz", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OSS-Fuzz-derived environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "open-source software targets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic cybersecurity tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_oss_fuzz", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 53, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_oss_fuzz" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "live Ubuntu virtual machine with verified tasks" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "OSWorld-Verified task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "mouse and keyboard actions" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 221, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_programbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-agent and multi-agent harnesses" }, "prompting": { "status": "disclosed", "value": "adaptive thinking" }, "sample_selection": { "status": "disclosed", "value": "tasks whose reference binaries passed hidden-suite quality filtering" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell and file-editing tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_programbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 197, section; multi-agent results on" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_programbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_proteingym_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ProteinGym" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "hard subset of mutant protein sequences" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_proteingym_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 234, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_proteingym_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "natural-language interface instructions" }, "sample_selection": { "status": "disclosed", "value": "professional desktop application screenshots" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "tool-free and tool-enabled conditions" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_screenspot_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 220, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_securebio_dna_synthesis_screening_evasion", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_securebio_dna_synthesis_screening_evasion", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 19, section, biological-risk evaluation list" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_securebio_dna_synthesis_screening_evasion" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_securebio_virology_capability_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_securebio_virology_capability_test", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 19, section, biological-risk evaluation list" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_securebio_virology_capability_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_shade_arena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SHADE-Arena" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 132, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_shade_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_super_agent_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agent tools implied" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_super_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Collaborating with Opus, early-tester quotation" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_super_agent_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260528_introducing_claude_opus_4_8", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260528_introducing_claude_opus_4_8", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual software-engineering tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 195, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "actively maintained repositories with larger multi-file changes" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 195, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "human-verified solvable subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 195, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench_2_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Harbor with the official Terminus-2 integration" }, "prompting": { "status": "disclosed", "value": "adaptive thinking" }, "sample_selection": { "status": "disclosed", "value": "public benchmark tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "terminal tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_terminal_bench_2_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 196, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_terminal_bench_2_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_toolathlon", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "execution-based checkers" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "tasks seeded from authentic application state" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-specific application tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_toolathlon", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 226, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_toolathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_usamo_2026", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "proof grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "official competition problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_usamo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 198, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_usamo_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vending_bench_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vending-Bench 2 simulation" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "simulated business-operation tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_vending_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 225, section" }, "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260528_introducing_claude_opus_4_8_benchmark_vending_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "release_id": "release_anthropic_20260528_introducing_claude_opus_4_8", "review_status": "verified", "source_id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_revision_id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arxivmath", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MathArena evaluation" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "recent self-contained paper-derived problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_arxivmath", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 260, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_arxivmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_automation_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "AutomationBench" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "tasks seeded from real workflow patterns" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "business-application tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_automationbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 295, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_automationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_benchcad", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "BenchCAD" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "CAD tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "computer-aided-design tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_benchcad", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 281, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_benchcad" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "ambiguous questions include an unknown-answer condition" }, "sample_selection": { "status": "disclosed", "value": "standard benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no tools disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_bias_benchmark_for_question_answering", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 83, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_bias_benchmark_for_question_answering" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biomysterybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "BioMysteryBench" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "biological analysis tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "computational biology tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 303, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_biomysterybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_blueprint_bench_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Blueprint-Bench 2" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "vision tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_blueprint_bench_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 279, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_blueprint_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-agent and multi-agent harnesses" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after the disclosed blocklist" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, programmatic tool calling, and code execution" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 266, section; multi-agent results on" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_chartmuseum", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "expert-annotated chart questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_chartmuseum", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 284, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_chartmuseum" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_chartqapro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world charts and questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_chartqapro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 283, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_chartqapro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "scientific charts from papers" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "tool-free and tool-enabled conditions" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 287, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_critpt", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_critpt", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 261, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_critpt" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cursorbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external evaluator harness" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cursorbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 257, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cursorbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberGym" }, "prompting": { "status": "disclosed", "value": "high-level weakness description supplied" }, "sample_selection": { "status": "disclosed", "value": "real open-source projects" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic software-development tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 61, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cyscenariobench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "benchmark challenges" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "cybersecurity tools implied" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cyscenariobench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "heading", "value": "Footnotes, cyber evaluation metrics note" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_cyscenariobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_deep_research_accuracy_completeness_and_objectivity", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "curated deep-research tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web-research and code-execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_deep_research_accuracy_completeness_and_objectivity", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 268, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_deep_research_accuracy_completeness_and_objectivity" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_deepsearchqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multi-step information-seeking prompts" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web-research tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 266, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_deepsearchqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_exploitbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "vulnerable software environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "patched vulnerabilities" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-development and execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_exploitbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 58, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_exploitbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_finance_agent", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external evaluator harness" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "financial research tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "financial-document research tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_finance_agent", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 290, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_finance_agent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_firefox_147_exploit_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "browser vulnerability environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "patched browser vulnerabilities" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-development and execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_firefox_147_exploit_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 62, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_firefox_147_exploit_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_frontierbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools implied" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontierbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "heading", "value": "Early feedback for Claude Fable 5, customer quotation" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontierbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_frontiercode", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external coding harness" }, "prompting": { "status": "disclosed", "value": "effort level varied" }, "sample_selection": { "status": "disclosed", "value": "Diamond subset where reported" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontiercode", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 255, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontiercode" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_frontierswe", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort unless otherwise noted" }, "sample_selection": { "status": "disclosed", "value": "open-ended long-horizon engineering problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontierswe", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 257, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_frontierswe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gdp_pdf", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "GDP.pdf" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "PDF-based tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document and vision tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gdp_pdf", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 277, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gdp_pdf" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gdpval_aa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external evaluation framework" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world professional tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-dependent professional tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gdpval_aa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 294, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gdpval_aa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_global_mmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_global_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 300, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_global_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "Diamond subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gpqa_diamond", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 258, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_graphwalks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "GraphWalks" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "long-context graph variants" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_graphwalks", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 262, section; named rows in evaluation table" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_graphwalks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_healthadminbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "healthcare-administration tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthadminbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 298, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthadminbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_healthbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "rubric-based grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "healthcare conversations" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 297, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_healthbench_professional", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "rubric-based grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "physician-authored conversations" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthbench_professional", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 297, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_healthbench_professional" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_hebbia_finance_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document, chart, and table analysis implied" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_hebbia_finance_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "heading", "value": "Evaluating Claude Fable 5 and Claude Mythos 5, Knowledge work paragraph" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_hebbia_finance_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after the disclosed blocklist" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "both without tools and with tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_humanity_s_last_exam", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 264, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_include", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 302, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_lab_bench_figqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LAB-Bench" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "biology-research figures" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_lab_bench_figqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 285, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_lab_bench_figqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_labbench2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LABBench2" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "biology research questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "research and retrieval tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_labbench2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 304, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_labbench2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_latchbio_bioinformatics", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LatchBio environment" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world bioinformatics problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bioinformatics tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_latchbio_bioinformatics", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 303, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_latchbio_bioinformatics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_legal_agent_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Legal Agent Benchmark" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "public and held-out legal task sets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document and spreadsheet tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_legal_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 292, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_legal_agent_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MCP-Atlas" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world tool workflows" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Model Context Protocol tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 293, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_minimal_linuxbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Minimal-LinuxBench" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Linux command-line tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_minimal_linuxbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 204, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_minimal_linuxbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mask", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "MASK dataset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_model_alignment_between_statements_and_knowledge", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 143, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_model_alignment_between_statements_and_knowledge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_multi_task_indic_language_understanding_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "Indic-language benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_multi_task_indic_language_understanding_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 301, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_multi_task_indic_language_understanding_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_officeqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OfficeQA" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "historical document corpus" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document search and reasoning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 289, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_officeqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_oss_fuzz", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OSS-Fuzz-derived environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "open-source software targets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic cybersecurity tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_oss_fuzz", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 60, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_oss_fuzz" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "verified OSWorld environment" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "OSWorld-Verified task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "mouse, keyboard, and vision-based computer interaction" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 280, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_programbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-agent and multi-agent harnesses" }, "prompting": { "status": "disclosed", "value": "adaptive thinking" }, "sample_selection": { "status": "disclosed", "value": "quality-filtered program-reconstruction tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell and file-editing tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_programbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 257, section; multi-agent results on" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_programbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_proteingym_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ProteinGym" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "hard mutant-protein subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_proteingym_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 304, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_proteingym_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_real_world_finance_v1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world finance tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "financial-analysis tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_real_world_finance_v1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 291, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_real_world_finance_v1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_real_world_finance_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world finance tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "financial-analysis tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_real_world_finance_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 290, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_real_world_finance_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_riemannbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_riemannbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 259, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_riemannbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "natural-language interface instructions" }, "sample_selection": { "status": "disclosed", "value": "professional desktop screenshots" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_screenspot_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 288, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_shade_arena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SHADE-Arena" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 202, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_shade_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort unless otherwise noted" }, "sample_selection": { "status": "disclosed", "value": "multilingual software-engineering tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 253, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multimodal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal multimodal SWE-bench harness" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort unless otherwise noted" }, "sample_selection": { "status": "disclosed", "value": "tasks with visual issue context" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent and image-input tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 253, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_multimodal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort unless otherwise noted" }, "sample_selection": { "status": "disclosed", "value": "actively maintained repositories" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 253, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort unless otherwise noted" }, "sample_selection": { "status": "disclosed", "value": "human-verified solvable subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 253, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench_2_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "mini-SWE-agent on a GKE cluster" }, "prompting": { "status": "disclosed", "value": "high effort" }, "sample_selection": { "status": "disclosed", "value": "unique public benchmark tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "terminal tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_terminal_bench_2_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 254, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_terminal_bench_2_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_toolathlon", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "execution-based checkers" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "tasks seeded from authentic application state" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-specific application tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_toolathlon", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 294, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_toolathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_usamo_2026", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "proof grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "official competition problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_usamo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 259, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_usamo_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vending_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Vending-Bench simulation" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "simulated business-operation tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_vending_bench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "page", "value": "page 293, section" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_vending_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_revision_id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_vibench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "base use cases" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "application-building tools implied" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_vibench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "locator": { "kind": "heading", "value": "Early feedback for Claude Fable 5, customer quotation" }, "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260609_claude_fable_5_and_claude_mythos_5_benchmark_vibench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "release_id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aa_briefcase", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external evaluation framework" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-dependent professional tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_aa_briefcase", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 137, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_aa_briefcase" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arxivmath", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MathArena evaluation" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "recent self-contained paper-derived problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_arxivmath", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 120, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_arxivmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_automation_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "AutomationBench" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "tasks seeded from real workflow patterns" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "business-application tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_automationbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 136, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_automationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_benchcad", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "BenchCAD" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "CAD tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "computer-aided-design tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_benchcad", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 127, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_benchcad" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "ambiguous questions include an unknown-answer condition" }, "sample_selection": { "status": "disclosed", "value": "standard benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no tools disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_bias_benchmark_for_question_answering", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 50, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_bias_benchmark_for_question_answering" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biomysterybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "BioMysteryBench" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "biological analysis tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "computational biology tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 142, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_biomysterybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "context compaction with programmatic tool calling" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after the disclosed blocklist" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, programmatic tool calling, and code execution" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 123, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_chartmuseum", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "expert-annotated chart questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_chartmuseum", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 128, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_chartmuseum" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "scientific charts from papers" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "tool-free and tool-enabled conditions" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_charxiv_reasoning", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 129, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cursorbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external evaluator harness" }, "prompting": { "status": "disclosed", "value": "effort level varied" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cursorbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 118, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cursorbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberGym" }, "prompting": { "status": "disclosed", "value": "high-level weakness description supplied" }, "sample_selection": { "status": "disclosed", "value": "real open-source projects" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic software-development tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cybergym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 32, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_exploitbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "vulnerable software environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "patched vulnerabilities" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-development and execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_exploitbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 30, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_exploitbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_firefox_147_exploit_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "browser vulnerability environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "patched browser vulnerabilities" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-development and execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_firefox_147_exploit_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 34, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_firefox_147_exploit_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_frontiercode_v1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external coding harness" }, "prompting": { "status": "disclosed", "value": "effort level varied" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_frontiercode_v1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 117, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_frontiercode_v1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gdp_pdf", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "GDP.pdf" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "PDF-based tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document and vision tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdp_pdf", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 124, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdp_pdf" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gdpval_aa_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external evaluation framework" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world professional tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-dependent professional tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdpval_aa_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 134, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_gdpval_aa_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_global_mmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_global_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 140, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_global_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_healthbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "rubric-based grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "healthcare conversations" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_healthbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 137, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_healthbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_healthbench_professional", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "rubric-based grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "physician-authored conversations" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_healthbench_professional", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 138, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_healthbench_professional" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after the disclosed blocklist" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "both without tools and with tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_humanity_s_last_exam", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 122, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_include", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 142, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_latchbio_bioinformatics", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LatchBio environment" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world bioinformatics problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bioinformatics tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_latchbio_bioinformatics", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 143, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_latchbio_bioinformatics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_legal_agent_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Legal Agent Benchmark" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "public and held-out legal task sets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document and spreadsheet tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_legal_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 133, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_legal_agent_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_linuxarena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LinuxArena" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Linux command-line tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_linuxarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 96, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_linuxarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mask", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "MASK dataset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_model_alignment_between_statements_and_knowledge", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 90, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_model_alignment_between_statements_and_knowledge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_multi_task_indic_language_understanding_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "Indic-language benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_multi_task_indic_language_understanding_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 141, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_multi_task_indic_language_understanding_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_officeqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OfficeQA" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "historical document corpus" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document search and reasoning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 131, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_officeqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_oss_fuzz", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OSS-Fuzz-derived environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "open-source software targets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic cybersecurity tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_oss_fuzz", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 31, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_oss_fuzz" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "verified OSWorld environment" }, "prompting": { "status": "disclosed", "value": "effort level varied" }, "sample_selection": { "status": "disclosed", "value": "OSWorld-Verified task set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "mouse, keyboard, and vision-based computer interaction" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_osworld_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 126, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_programbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ProgramBench agent harness" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "quality-filtered program-reconstruction tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell and file-editing tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_programbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 121, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_programbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_proteingym_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ProteinGym" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "hard mutant-protein subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_proteingym_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 143, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_proteingym_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_real_world_finance_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world finance tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "financial-analysis tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_real_world_finance_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 132, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_real_world_finance_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_shade_arena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SHADE-Arena" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 95, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_shade_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "standard SWE-bench configuration" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual software-engineering tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 116, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multimodal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal multimodal SWE-bench harness" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "tasks with screenshots or design mockups" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent and image-input tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 116, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_multimodal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "standard SWE-bench configuration" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "actively maintained repositories" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 116, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "standard SWE-bench configuration" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "human-verified solvable subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 116, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_terminal_bench_2_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "mini-SWE-agent" }, "prompting": { "status": "disclosed", "value": "adaptive thinking" }, "sample_selection": { "status": "disclosed", "value": "public benchmark tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "terminal tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_terminal_bench_2_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 116, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_terminal_bench_2_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_toolathlon", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "execution-based checkers" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "tasks seeded from authentic application state" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-specific application tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_toolathlon", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 134, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_toolathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_usamo_2026", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "proof grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "official competition problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_usamo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "locator": { "kind": "page", "value": "page 119, section" }, "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260630_introducing_claude_sonnet_5_benchmark_usamo_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "release_id": "release_anthropic_20260630_introducing_claude_sonnet_5", "review_status": "verified", "source_id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_revision_id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aa_briefcase", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external evaluation framework" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-dependent professional tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_aa_briefcase", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 182, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_aa_briefcase" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aa_coding_agent_index", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "effort level varied" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools implied" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_aa_coding_agent_index", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Performance and cost-effectiveness, coding chart tab" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_aa_coding_agent_index" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260724_introducing_claude_opus_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260724_introducing_claude_opus_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC-AGI evaluation harness" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "ARC-AGI-1 tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 184, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC-AGI evaluation harness" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "ARC-AGI-2 tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 184, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arc_agi_3", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ARC-AGI-3 environment" }, "prompting": { "status": "disclosed", "value": "high effort in the summary table" }, "sample_selection": { "status": "disclosed", "value": "ARC-AGI-3 tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "interactive environment tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_3", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 186, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arc_agi_3" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_arxivmath", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MathArena evaluation" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "recent self-contained paper-derived problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arxivmath", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 158, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_arxivmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_automation_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "AutomationBench" }, "prompting": { "status": "disclosed", "value": "effort level varied" }, "sample_selection": { "status": "disclosed", "value": "tasks seeded from real workflow patterns" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "business-application tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_automationbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 183, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_automationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_benchcad", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "BenchCAD" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "CAD tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "computer-aided-design tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_benchcad", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 175, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_benchcad" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "ambiguous questions include an unknown-answer condition" }, "sample_selection": { "status": "disclosed", "value": "standard benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "no tools disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_bias_benchmark_for_question_answering", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 65, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_bias_benchmark_for_question_answering" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_biomysterybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "BioMysteryBench" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "biological analysis tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "computational biology tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_biomysterybench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 192, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_biomysterybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-agent and multi-agent harnesses with context compaction" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after the disclosed blocklist" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web search, web fetch, programmatic tool calling, and code execution" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_browsecomp", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 162, section; multi-agent results on" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_chartography", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Chartography" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "chart tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "vision and chart-analysis tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_chartography", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 173, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_chartography" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cursorbench_3_2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "effort level varied" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_cursorbench_3_2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "heading", "value": "Performance and cost-effectiveness, coding evaluation paragraph and chart tab" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_cursorbench_3_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_launch_anthropic_20260724_introducing_claude_opus_5", "source_revision_id": "revision_anthropic_revision_launch_anthropic_20260724_introducing_claude_opus_5", "source_type": "launch_page", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_cyscenariobench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyScenarioBench" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "benchmark challenges" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "cybersecurity tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_cyscenariobench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 41, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_cyscenariobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_deep_research_accuracy_completeness_and_objectivity", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "curated deep-research tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web-research and code-execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deep_research_accuracy_completeness_and_objectivity", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 165, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deep_research_accuracy_completeness_and_objectivity" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_deepsearchqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multi-step information-seeking prompts" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "web-research tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deepsearchqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 164, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deepsearchqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_deepswe_v1_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "DeepSWE" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "diverse tasks written specifically for the evaluation" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deepswe_v1_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 153, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_deepswe_v1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_exploitbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "vulnerable software environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "patched vulnerabilities" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-development and execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_exploitbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 37, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_exploitbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_exploitgym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ExploitGym" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "cybersecurity and software-development tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_exploitgym", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 42, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_exploitgym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_firefox_147_exploit_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "browser vulnerability environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "patched browser vulnerabilities" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "software-development and execution tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_firefox_147_exploit_evaluation", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 40, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_firefox_147_exploit_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_frontier_bench_v0_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Harbor evaluation using mini-SWE-agent and a GKE backend" }, "prompting": { "status": "disclosed", "value": "effort level varied" }, "sample_selection": { "status": "disclosed", "value": "public benchmark tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_frontier_bench_v0_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 156, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_frontier_bench_v0_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_frontiercode_v1_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Cognition evaluation harness" }, "prompting": { "status": "disclosed", "value": "single issue description; effort level varied" }, "sample_selection": { "status": "disclosed", "value": "main and extended task sets derived from real pull requests" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools in a containerized repository" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_frontiercode_v1_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 155, section, figure .A and adjacent text" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_frontiercode_v1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gdp_pdf", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "GDP.pdf" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "PDF-based tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document and vision tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_gdp_pdf", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 178, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_gdp_pdf" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_gdpval_aa_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "external evaluation framework" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world professional tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-dependent professional tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_gdpval_aa_v2", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 182, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_gdpval_aa_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_global_mmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_global_mmlu", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 190, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_global_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_healthbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "rubric-based grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "healthcare conversations" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_healthbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 188, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_healthbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_healthbench_professional", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "rubric-based grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "physician-authored conversations" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_healthbench_professional", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 189, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_healthbench_professional" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "benchmark questions after the disclosed blocklist" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "both without tools and with tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_humanity_s_last_exam", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 160, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_imo_2026", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "proof grading" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "official competition problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_imo_2026", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 157, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_imo_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_include", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 191, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_latchbio_bioinformatics", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LatchBio environment" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world bioinformatics problems" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "bioinformatics tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_latchbio_bioinformatics", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 192, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_latchbio_bioinformatics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_legal_agent_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Legal Agent Benchmark" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "legal task sets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document and spreadsheet tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_legal_agent_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 181, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_legal_agent_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_linuxarena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "LinuxArena" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Linux command-line tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_linuxarena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 122, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_linuxarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MCP-Atlas" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "real-world tool workflows" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Model Context Protocol tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_mcp_atlas", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 181, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_mask", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "MASK dataset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_model_alignment_between_statements_and_knowledge", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 112, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_model_alignment_between_statements_and_knowledge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_multi_task_indic_language_understanding_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "Indic-language benchmark questions" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_multi_task_indic_language_understanding_benchmark", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 191, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_multi_task_indic_language_understanding_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_officeqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OfficeQA" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "historical document corpus" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "document search and reasoning tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_officeqa", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 180, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_officeqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_oss_fuzz", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OSS-Fuzz-derived environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "open-source software targets" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic cybersecurity tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_oss_fuzz", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 38, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_oss_fuzz" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_osworld_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "effort level varied" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "mouse, keyboard, and vision-based computer interaction" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_osworld_2_0", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 177, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_osworld_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_programbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "single-agent and multi-agent harnesses" }, "prompting": { "status": "disclosed", "value": "adaptive thinking" }, "sample_selection": { "status": "disclosed", "value": "quality-filtered program-reconstruction tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell and file-editing tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_programbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 159, section; multi-agent results on" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_programbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_proteingym_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "ProteinGym" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "hard mutant-protein subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_proteingym_hard", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 193, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_proteingym_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_riemannbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_riemannbench", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 158, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_riemannbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_shade_arena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SHADE-Arena" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "agentic tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_shade_arena", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 121, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_shade_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "standard SWE-bench configuration" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "multilingual software-engineering tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_multilingual", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 153, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_multimodal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "SWE-bench Multimodal test harness" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "tasks with screenshots or design mockups" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent and image-input tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_multimodal", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 153, section, variant list; harness details in appendix section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_multimodal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "standard SWE-bench configuration" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "actively maintained repositories" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_pro", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 153, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "standard SWE-bench configuration" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "human-verified solvable subset" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "coding-agent tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 153, section, variant list" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_toolathlon_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "execution-based verified checkers" }, "prompting": { "status": "disclosed", "value": "adaptive thinking at max effort" }, "sample_selection": { "status": "disclosed", "value": "verified tool-use tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "task-specific application tools" } }, "id": "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_toolathlon_verified", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "locator": { "kind": "page", "value": "page 182, section" }, "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "provenance": { "record_ids": [ "occurrence_anthropic_occurrence_anthropic_20260724_introducing_claude_opus_5_benchmark_toolathlon_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "release_id": "release_anthropic_20260724_introducing_claude_opus_5", "review_status": "verified", "source_id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_revision_id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "source_type": "system_card", "summary": "The first-party source identifies this benchmark for an Anthropic model or model set in the release." }, { "benchmark_id": "benchmark_aider_polyglot", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_aider_polyglot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_bfcl_v3_multiturn", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_bfcl_v3_multiturn", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_bfcl_v3_multiturn" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cnmo_2024", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_cnmo_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_frames", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_frames" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hmmt_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_hmmt_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_hmmt_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_humanity_s_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source names the framework used for the software-engineering benchmark." }, "prompting": { "status": "disclosed", "value": "The source discloses a shared sampling configuration." }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source identifies tool-mediated evaluation where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_0528_tau_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Evaluation Results, DeepSeek-R1-0528 table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1_0528", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_0528_tau_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "release_id": "release_deepseek_20250528_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_hf_r1_0528", "source_revision_id": "revision_deepseek_revision_source_hf_r1_0528", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aider_polyglot", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_aider_polyglot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_alpacaeval_2_0", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_alpacaeval_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_c_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_c_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_c_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cluewsc", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_cluewsc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cnmo_2024", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_cnmo_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_frames", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_frames" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_lite_preview_aime", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Launch page heading and named benchmark sentence" }, "model_id": "model_deepseek_model_deepseek_r1_lite_preview", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_lite_preview_aime" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-20", "release_id": "release_deepseek_20241120_deepseek_r1_lite_preview", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20241120_deepseek_r1_lite_preview", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20241120_deepseek_r1_lite_preview", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_lite_preview_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "Launch page heading and named benchmark sentence" }, "model_id": "model_deepseek_model_deepseek_r1_lite_preview", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_lite_preview_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-20", "release_id": "release_deepseek_20241120_deepseek_r1_lite_preview", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20241120_deepseek_r1_lite_preview", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20241120_deepseek_r1_lite_preview", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_math_500", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_math_500" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_mmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies benchmark-specific evaluation frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states prompt sources and benchmark-specific prompt adaptations." }, "sample_selection": { "status": "disclosed", "value": "The report identifies selected benchmark windows or contest sets where applicable." }, "shot_configuration": { "status": "disclosed", "value": "The report states zero-shot or original-protocol use where applicable." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_r1_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "locator": { "kind": "row", "value": "paper page twelve, DeepSeek-R1 Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_r1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_r1_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "release_id": "release_deepseek_20250120_deepseek_reasoner", "review_status": "verified", "source_id": "source_deepseek_source_report_r1", "source_revision_id": "revision_deepseek_revision_source_report_r1", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_0517_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "locator": { "kind": "row", "value": "Change Log, dated heading, instruction-following benchmark line" }, "model_id": "model_deepseek_model_deepseek_v2_0517", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_0517_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-17", "release_id": "release_deepseek_20240517_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20240517_deepseek_chat", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20240517_deepseek_chat", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_0628_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_0628", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_0628_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "release_id": "release_deepseek_20240628_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20240628_deepseek_chat", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20240628_deepseek_chat", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_0628_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_0628", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_0628_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "release_id": "release_deepseek_20240628_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20240628_deepseek_chat", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20240628_deepseek_chat", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_0628_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_0628", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_0628_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "release_id": "release_deepseek_20240628_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20240628_deepseek_chat", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20240628_deepseek_chat", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_0628_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_0628", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_0628_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "release_id": "release_deepseek_20240628_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20240628_deepseek_chat", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20240628_deepseek_chat", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_1210_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, capability list item keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5_1210", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_1210_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25_1210", "source_revision_id": "revision_deepseek_revision_source_hf_v25_1210", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_math_500", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_1210_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, capability list item keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5_1210", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_1210_math_500" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "release_id": "release_deepseek_20241210_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25_1210", "source_revision_id": "revision_deepseek_revision_source_hf_v25_1210", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aider", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_aider", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_aider" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_alignbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_alignbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_alignbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_alpacaeval_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_alpacaeval_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_ds_arena_code", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_ds_arena_code", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_ds_arena_code" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_ds_fim_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_ds_fim_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_ds_fim_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_humaneval_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_humaneval_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_humaneval_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_mt_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v2_5_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "locator": { "kind": "row", "value": "Introduction, model comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v2_5", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v2_5_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "release_id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "review_status": "verified", "source_id": "source_deepseek_source_hf_v25", "source_revision_id": "revision_deepseek_revision_source_hf_v25", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_0324_aime", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Reasoning Capabilities, benchmark list item keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_0324", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_0324_aime" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_hf_v3_0324", "source_revision_id": "revision_deepseek_revision_source_hf_v3_0324", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_0324_gpqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Reasoning Capabilities, benchmark list item keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_0324", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_0324_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_hf_v3_0324", "source_revision_id": "revision_deepseek_revision_source_hf_v3_0324", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_0324_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Reasoning Capabilities, benchmark list item keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_0324", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_0324_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_hf_v3_0324", "source_revision_id": "revision_deepseek_revision_source_hf_v3_0324", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_0324_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Reasoning Capabilities, benchmark list item keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_0324", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_0324_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "release_id": "release_deepseek_20250324_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_hf_v3_0324", "source_revision_id": "revision_deepseek_revision_source_hf_v3_0324", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aider_polyglot", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_aider_polyglot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_browsecomp_zh", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_browsecomp_zh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_browsecomp_zh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hmmt_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_hmmt_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_hmmt_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_humanity_s_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies internal search or code-agent frameworks where applicable." }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes search and code agent evaluations." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminal_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "release_id": "release_deepseek_20250821_deepseek_v3_1", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31", "source_revision_id": "revision_deepseek_revision_source_hf_v31", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aider_polyglot", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_aider_polyglot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_browsecomp_zh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_browsecomp_zh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_browsecomp_zh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_humanity_s_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_terminal_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, comparison table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_1_terminus_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_deepseek_20250922_deepseek_v3_1_terminus", "review_status": "verified", "source_id": "source_deepseek_source_hf_v31_terminus", "source_revision_id": "revision_deepseek_revision_source_hf_v31_terminus", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_browsecomp_zh", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_browsecomp_zh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_browsecomp_zh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aider_polyglot", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_aider_polyglot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_browsecomp_zh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_browsecomp_zh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_browsecomp_zh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hmmt_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_hmmt_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_hmmt_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_humanity_s_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluations without tools from agentic tool use." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_exp_terminal_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Introduction, public benchmark table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_exp_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "release_id": "release_deepseek_20250929_deepseek_v3_2_exp", "review_status": "verified", "source_id": "source_deepseek_source_hf_v32_exp", "source_revision_id": "revision_deepseek_revision_source_hf_v32_exp", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hmmt_feb_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_hmmt_feb_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_hmmt_feb_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hmmt_nov_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_hmmt_nov_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_hmmt_nov_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_humanity_s_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_imoanswerbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mcp_mark", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_mcp_mark", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_mcp_mark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mcp_universe", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_mcp_universe", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_mcp_universe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_aime_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cmo_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_cmo_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_cmo_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hmmt_feb_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_hmmt_feb_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_hmmt_feb_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hmmt_nov_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_hmmt_nov_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_hmmt_nov_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_humanity_s_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_icpc_wf_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_icpc_wf_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_icpc_wf_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_imo_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_imo_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_imo_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_imoanswerbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_ioi_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_ioi_2025", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_ioi_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "The report states the applicable reasoning-task prompt protocol." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset or competition problem set where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "locator": { "kind": "row", "value": "paper Speciale results tables, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_speciale_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2_speciale", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_tau2_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_tau2_bench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_tau2_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_terminal_bench_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_tool_decathlon", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The report identifies internal or named agent frameworks where applicable." }, "prompting": { "status": "disclosed", "value": "The report states a benchmark prompt template where applicable." }, "sample_selection": { "status": "disclosed", "value": "The report identifies the text-only subset where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The report states function-call or agent-tool use where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_2_tool_decathlon", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "paper page thirteen, Main Results table, row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3_2", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_2_tool_decathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "release_id": "release_deepseek_20251201_deepseek_v3_2", "review_status": "verified", "source_id": "source_deepseek_source_report_v32", "source_revision_id": "revision_deepseek_revision_source_report_v32", "source_type": "technical_report", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_agieval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_agieval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aider_edit", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_aider_edit", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_aider_edit" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aider_polyglot", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_aider_polyglot", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_aider_polyglot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_aime_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_alpacaeval_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_alpacaeval_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_alpacaeval_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_arc_challenge", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_arc_easy", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_arc_easy", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_arc_easy" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_arena_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_c3", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_c3", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_c3" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_c_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_c_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_c_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_ccpm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_ccpm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_ccpm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cluewsc", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cluewsc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cmath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cmrc", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_cmrc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cmrc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cnmo_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_cnmo_2024", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cnmo_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cruxeval_i", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_cruxeval_i", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cruxeval_i" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cruxeval_o", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_cruxeval_o", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_cruxeval_o" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_frames", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_frames", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_frames" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_humaneval_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_humaneval_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_humaneval_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_ifeval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench_base", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_livecodebench_base", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_livecodebench_base" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_longbench_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_longbench_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_math_500", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_math_500", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_math_500" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_mbpp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_mmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmmlu_non_english", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_mmmlu_non_english", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_mmmlu_non_english" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_naturalquestions", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_naturalquestions", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_naturalquestions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_needle_in_a_haystack", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_needle_in_a_haystack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_needle_in_a_haystack" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_pile_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_pile_test", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_pile_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_piqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_piqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_race_high", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_race_high", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_race_high" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_race_middle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_race_middle", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_race_middle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_triviaqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_triviaqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source states a repeated-testing rule for smaller benchmark sets." }, "shot_configuration": { "status": "disclosed", "value": "The source table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v3_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "locator": { "kind": "row", "value": "Evaluation Results, source table or figure entry keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v3", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v3_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "release_id": "release_deepseek_20241226_deepseek_chat", "review_status": "verified", "source_id": "source_deepseek_source_repo_v3", "source_revision_id": "revision_deepseek_revision_source_repo_v3", "source_type": "official_repository", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_agieval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_agieval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_agieval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_apex", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_apex", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_apex" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_apex_shortlist", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_apex_shortlist", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_apex_shortlist" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_bbh", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_bigcodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_bigcodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_bigcodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_browsecomp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_c_eval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_chinese_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_chinese_simpleqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_chinese_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cluewsc", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_cluewsc", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_cluewsc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cmath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_cmath", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_cmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_cmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_cmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_codeforces", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_corpusqa_1m", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_corpusqa_1m", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_corpusqa_1m" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_drop", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_facts_parametric", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_facts_parametric", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_facts_parametric" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gdpval_aa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_gdpval_aa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_gdpval_aa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_gpqa_diamond", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_gsm8k", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_hellaswag", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hmmt_2026_feb", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_hmmt_2026_feb", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_hmmt_2026_feb" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_humaneval", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_humanity_s_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_imoanswerbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_livecodebench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_longbench_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_longbench_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_longbench_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_math", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mcpatlas", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_mcpatlas", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mcpatlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mcpatlas_public", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_mcpatlas_public", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mcpatlas_public" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_mgsm", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu_redux", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_mmmlu", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_mrcr_1m", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_mrcr_1m", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_mrcr_1m" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_multiloko", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_multiloko", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_multiloko" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_simpleqa_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_simpleqa_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_simpleqa_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_supergpqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_swe_bench_multilingual", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_swe_bench_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_swe_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_swe_pro", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_swe_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_terminal_bench_2_0", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_toolathlon", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_toolathlon", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_toolathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_triviaqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_triviaqa", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_triviaqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "The base-model table identifies the applicable shot regime." }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_family_winogrande", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Evaluation Results, source table row keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_family", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_family_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "release_id": "release_deepseek_20260424_deepseek_v4", "review_status": "verified", "source_id": "source_deepseek_source_hf_v4", "source_revision_id": "revision_deepseek_revision_source_hf_v4", "source_type": "official_hugging_face_card", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_agents_last_exam", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies public and internal benchmark-set scope." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_agents_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_agents_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_automation_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies public and internal benchmark-set scope." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_automationbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_automationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies public and internal benchmark-set scope." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_cybergym", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_deepswe", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies public and internal benchmark-set scope." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_deepswe", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_deepswe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_dsbench_fullstack", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies public and internal benchmark-set scope." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_dsbench_fullstack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_dsbench_fullstack" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_dsbench_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies public and internal benchmark-set scope." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_dsbench_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_dsbench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies public and internal benchmark-set scope." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_nl2repo", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_terminal_bench_2_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies public and internal benchmark-set scope." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_terminal_bench_2_1", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_terminal_bench_2_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_toolathlon_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies public and internal benchmark-set scope." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_toolathlon_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_0731_toolathlon_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "release_id": "release_deepseek_20260731_deepseek_v4_flash_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260731_deepseek_v4_flash_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_agents_last_exam", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies text-only or multimodal benchmark scope where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_agents_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_agents_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_apexbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies text-only or multimodal benchmark scope where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_apexbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_apexbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_automation_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies text-only or multimodal benchmark scope where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_automationbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_automationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_chartography", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies text-only or multimodal benchmark scope where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_chartography", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_chartography" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_deepswe", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies text-only or multimodal benchmark scope where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_deepswe", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_deepswe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_dsbench_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies text-only or multimodal benchmark scope where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_dsbench_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_dsbench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies text-only or multimodal benchmark scope where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_nl2repo", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_terminal_bench_2_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies text-only or multimodal benchmark scope where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_terminal_bench_2_1", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_terminal_bench_2_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_zerobench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "The source identifies the DeepSeek Harness minimal mode." }, "prompting": { "status": "disclosed", "value": "The source states the maximum reasoning-effort evaluation mode." }, "sample_selection": { "status": "disclosed", "value": "The source identifies text-only or multimodal benchmark scope where applicable." }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_zerobench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_flash_vision_exp_zerobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "release_id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_agents_last_exam", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluation with and without tools where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_agents_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_agents_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_automation_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluation with and without tools where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_automationbench", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_automationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluation with and without tools where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_cybergym", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_deepswe", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluation with and without tools where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_deepswe", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_deepswe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_dsbench_fullstack", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluation with and without tools where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_dsbench_fullstack", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_dsbench_fullstack" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_dsbench_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluation with and without tools where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_dsbench_hard", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_dsbench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluation with and without tools where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_humanity_s_last_exam", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_humanity_s_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluation with and without tools where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_nl2repo", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_terminal_bench_2_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluation with and without tools where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_terminal_bench_2_1", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_terminal_bench_2_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_toolathlon_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes evaluation with and without tools where applicable." } }, "id": "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_toolathlon_verified", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "locator": { "kind": "row", "value": "Change Log, dated heading, benchmark line keyed by benchmark_id" }, "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "provenance": { "record_ids": [ "occurrence_deepseek_occurrence_deepseek_v4_pro_0813_toolathlon_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_deepseek_20260813_deepseek_v4_pro_update", "review_status": "verified", "source_id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_revision_id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "source_type": "launch_page", "summary": "The reviewed first-party source names this benchmark for DeepSeek's released model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240208_the_next_chapter_of_our_gemini_era_mmlu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "A new state of the art, paragraph 2" }, "model_id": "model_google_deepmind_google_deepmind_gemini_ultra_1_0", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240208_the_next_chapter_of_our_gemini_era_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-08", "release_id": "release_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Grammar and reference materials supplied in-context" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240215_our_next_generation_model_gemini_1_5_mtob", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Enhanced performance, paragraphs 2 and 3" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_pro_february_preview", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240215_our_next_generation_model_gemini_1_5_mtob" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "release_id": "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_needle_in_a_haystack", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240215_our_next_generation_model_gemini_1_5_niah", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Enhanced performance, paragraphs 2 and 3" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_pro_february_preview", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240215_our_next_generation_model_gemini_1_5_niah" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "release_id": "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_agieval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_agieval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_agieval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_arc_challenge", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arc_easy", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_arc_easy" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_bbq_ambiguous", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "1-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bbq_ambiguous", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 8, Table 8, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bbq_ambiguous" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_bbq_disambiguated", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bbq_disambiguated", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 8, Table 8, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bbq_disambiguated" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bigbench_hard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bigbench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_bold", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bold", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 8, Table 8, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_bold" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_boolq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_boolq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_commonsenseqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "7-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_commonsenseqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_commonsenseqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_crows_pairs", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_crows_pairs", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 8, Table 8, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_crows_pairs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_human_eval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_human_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "3-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_mbpp", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_mmlu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_natural_questions", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_natural_questions", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_natural_questions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_openbookqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_openbookqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_openbookqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_piqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_realtoxicity", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_realtoxicity", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 8, Table 8, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_realtoxicity" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_socialiqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_social_iqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_social_iqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_toxigen", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_toxigen", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 8, Table 8, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_toxigen" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_triviaqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_triviaqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_truthfulqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "MC2 evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_truthfulqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 8, Table 8, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_truthfulqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_winobias_type_1_2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winobias_type_1_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 8, Table 8, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winobias_type_1_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_winobias_type_2_2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winobias_type_2_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 8, Table 8, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winobias_type_2_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_winogender", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winogender", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 8, Table 8, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winogender" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Partial scoring" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winogrande", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "locator": { "kind": "table", "value": "Page 6, Table 6, named benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "release_id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_activitynet_qa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_activitynet_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_ai2d", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Transparent bounding-box protocol" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_ai2d", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_ai2d" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_amc_2022_23", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_amc_2022_2023", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_amc_2022_2023" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_automatic_speech_recognition_from_one_book", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_asrob", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Long-context evaluation section and named benchmark subsection" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_asrob" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_betterchartqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_better_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_better_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "3-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_bigbench_hard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_bigbench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_blink", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_blink" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_chartqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Chain-of-thought rationale" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_chemicaldiagramqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_chemical_diagram_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_chemical_diagram_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Google Cloud OCR annotations supplied" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "Variable-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_dude", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_dude", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_dude" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_egoschema", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Chain-of-thought rationale" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_egoschema", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_egoschema" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_functional_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_functional_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_functional_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "11-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "10-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hiddenmath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_hiddenmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Chat preamble" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_human_eval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_human_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_infographicvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_infographicvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Minerva prompt" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Chain-of-thought rationale" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "8-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mmlu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Chain-of-thought rationale" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_machine_translation_from_one_book", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mtob", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Long-context evaluation section and named benchmark subsection" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_mtob" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_natural2code", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Chat preamble" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_natural2code" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_needle_in_a_haystack", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_niah", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Long-context evaluation section and named benchmark subsection" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_niah" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_record_1h_videoqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_one_hour_videoqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Long-context evaluation section and named benchmark subsection" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_one_hour_videoqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_openeqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_openeqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_openeqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_physicsfinals", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_physics_finals", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_physics_finals" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_tat_dqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_tat_dqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_tat_dqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_textvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_textvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_vatex", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vatex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vatex" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_vatex_zh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vatex_zh", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vatex_zh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_vqav2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vqav2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_v_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vstar_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_vstar_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_wmt23", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "1-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_wmt23", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_wmt23" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_youcook2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_youcook2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 30, 39, and 40, benchmark tables and named rows" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte_youcook2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "release_id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_1_5_report", "source_revision_id": "revision_google_deepmind_revision_support_gemini_1_5_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hiddenmath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Improved overall quality, paragraph 2" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_hiddenmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Improved overall quality, paragraph 2" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Improved overall quality, paragraph 2" }, "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "release_id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_activitynet_qa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Sixteen frames sampled linearly" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_activitynet_qa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 22, Table 17, named video benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_activitynet_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_agieval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "3-to-5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_agieval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_agieval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Character-length normalization" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "25-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_arc_challenge", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arc_easy", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Character-length normalization" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_arc_easy" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_big_bench_extra_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bigbench_extra_hard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bigbench_extra_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bigbench_hard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bigbench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_bird_sql", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bird_sql", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_bird_sql" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_blink", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_blink", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_blink" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_boolq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Character-length normalization" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_boolq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_chartqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_chartqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_coco_caption", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_coco_caption", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_coco_caption" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_countbench_vqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_countbench_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_countbench_vqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_docvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "1-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_eclektic", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "First-line stripping" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "2-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_eclektic", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_eclektic" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_facts_grounding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_facts_grounding", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_facts_grounding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_flores", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "1-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_flores", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_flores" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_global_mmlu_lite", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_global_mmlu_lite", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_global_mmlu_lite" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Character-length normalization" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "10-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hiddenmath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_hiddenmath", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_hiddenmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_human_eval", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_human_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_infographicvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_infographicvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_infographicvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Chain-of-thought" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_lmarena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Blind side-by-side human preference evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 5, Table 5, Chatbot Arena evaluation" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_lmarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_math", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Sixteen frames sampled linearly" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mathvista", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 22, Table 17, named video benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "3-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mbpp", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "8-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mgsm", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmlu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "3-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_natural2code", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_natural2code", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_natural2code" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_natural_questions", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_natural_questions", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_natural_questions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_ok_vqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_ok_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_ok_vqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_perception_test_mcvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Sixteen frames sampled linearly" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_perception_test_mcvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 22, Table 17, named video benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_perception_test_mcvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_piqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Character-length normalization" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_realworldqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_simpleqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Pages 6 and 23, capability benchmark tables and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_socialiqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Character-length normalization" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_social_iqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_social_iqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_spatialsense_vqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_spatialsense_vqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_spatialsense_vqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_tallyqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_tallyqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_tallyqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_textvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_textvqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_textvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_triviaqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_triviaqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_vqav2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "4-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_vqav2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 25, Table 20, named vision benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_vqav2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Character-length normalization" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_winogrande", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_wmt24", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_wmt24_plus_plus", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_wmt24_plus_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_xquad", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_xquad", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Page 24, Table 19, named text benchmark row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250312_introducing_gemma_3_xquad" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250312_introducing_gemma_3", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_3_report", "source_revision_id": "revision_google_deepmind_revision_support_gemma_3_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "No majority voting or other test-time cost-increasing technique" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Enhanced reasoning and Advanced coding sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "No majority voting or other test-time cost-increasing technique" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_gpqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Enhanced reasoning and Advanced coding sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Tools disabled" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_humanitys_last_exam", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Enhanced reasoning and Advanced coding sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_lmarena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Enhanced reasoning and Advanced coding sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_lmarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Enhanced reasoning and Advanced coding sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Custom agent setup" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Enhanced reasoning and Advanced coding sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "release_id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_lmarena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250417_introducing_gemini_2_5_flash_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Our most cost-efficient thinking model, preceding paragraph" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250417_introducing_gemini_2_5_flash_lmarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250417_introducing_gemini_2_5_flash", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250417_introducing_gemini_2_5_flash", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_videomme", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_video_mme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Release description, paragraph 3" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_video_mme" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_webdev_arena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Release description, paragraph 3" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi_webdev_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_videomme", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_video_mme", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Best-in-class frontend web development and Video to code sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_video_mme" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_webdev_arena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Best-in-class frontend web development and Video to code sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc_webdev_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "release_id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_wmt24", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_wmt24_plus_plus", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "heading", "value": "Key Capabilities of Gemma 3n, multilingual capability item" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil_wmt24_plus_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_livecodebench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini Pro performs better than ever and Deep Think sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_lmarena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini Pro performs better than ever and Deep Think sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_lmarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini Pro performs better than ever and Deep Think sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_usamo_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_usamo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini Pro performs better than ever and Deep Think sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_usamo_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_webdev_arena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini Pro performs better than ever and Deep Think sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e_webdev_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "release_id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_lmarena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "heading", "value": "What is new in Gemma 3n and MatFormer sections" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_lmarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_mmlu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "heading", "value": "What is new in Gemma 3n and MatFormer sections" }, "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "release_id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_medqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai_medqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_medgemma_vision_language_general", "locator": { "kind": "heading", "value": "MedGemma: A multimodal generative model for health, model capability items" }, "model_id": "model_google_deepmind_google_deepmind_medgemma_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai_medqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "release_id": "release_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_drop", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "locator": { "kind": "heading", "value": "Towards better quality-efficiency trade-off and foundational capabilities sections" }, "model_id": "model_google_deepmind_google_deepmind_t5gemma_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "release_id": "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_gsm8k", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "locator": { "kind": "heading", "value": "Towards better quality-efficiency trade-off and foundational capabilities sections" }, "model_id": "model_google_deepmind_google_deepmind_t5gemma_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "release_id": "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_mmlu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "locator": { "kind": "heading", "value": "Towards better quality-efficiency trade-off and foundational capabilities sections" }, "model_id": "model_google_deepmind_google_deepmind_t5gemma_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "release_id": "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_superglue", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_superglue", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "locator": { "kind": "heading", "value": "Towards better quality-efficiency trade-off and foundational capabilities sections" }, "model_id": "model_google_deepmind_google_deepmind_t5gemma_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo_superglue" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "release_id": "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Best-of sampling disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_aime_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "table", "value": "Page 4, capability benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "release_id": "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_2_5_deep_think_card", "source_revision_id": "revision_google_deepmind_revision_support_gemini_2_5_deep_think_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Tools disabled" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_humanitys_last_exam", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "table", "value": "Page 4, capability benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "release_id": "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_2_5_deep_think_card", "source_revision_id": "revision_google_deepmind_revision_support_gemini_2_5_deep_think_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_imo_2025", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Single-sample result for the released model" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_imo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "table", "value": "Page 4, capability benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_imo_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "release_id": "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_2_5_deep_think_card", "source_revision_id": "revision_google_deepmind_revision_support_gemini_2_5_deep_think_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "Benchmark UI date range disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_livecodebench_v6", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "table", "value": "Page 4, capability benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250801_try_deep_think_in_the_gemini_app_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "release_id": "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "review_status": "verified", "source_id": "source_google_deepmind_support_gemini_2_5_deep_think_card", "source_revision_id": "revision_google_deepmind_revision_support_gemini_2_5_deep_think_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_arc_challenge", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "locator": { "kind": "table", "value": "Page 4, downstream benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "review_status": "verified", "source_id": "source_google_deepmind_support_vaultgemma_report", "source_revision_id": "revision_google_deepmind_revision_support_vaultgemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arc_easy", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_arc_easy", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "locator": { "kind": "table", "value": "Page 4, downstream benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_arc_easy" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "review_status": "verified", "source_id": "source_google_deepmind_support_vaultgemma_report", "source_revision_id": "revision_google_deepmind_revision_support_vaultgemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_boolq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_boolq", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "locator": { "kind": "table", "value": "Page 4, downstream benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_boolq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "review_status": "verified", "source_id": "source_google_deepmind_support_vaultgemma_report", "source_revision_id": "revision_google_deepmind_revision_support_vaultgemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_hellaswag", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "locator": { "kind": "table", "value": "Page 4, downstream benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "review_status": "verified", "source_id": "source_google_deepmind_support_vaultgemma_report", "source_revision_id": "revision_google_deepmind_revision_support_vaultgemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_piqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_piqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "locator": { "kind": "table", "value": "Page 4, downstream benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "review_status": "verified", "source_id": "source_google_deepmind_support_vaultgemma_report", "source_revision_id": "revision_google_deepmind_revision_support_vaultgemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_socialiqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "0-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_social_iqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "locator": { "kind": "table", "value": "Page 4, downstream benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_social_iqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "review_status": "verified", "source_id": "source_google_deepmind_support_vaultgemma_report", "source_revision_id": "revision_google_deepmind_revision_support_vaultgemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_triviaqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "disclosed", "value": "5-shot" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_triviaqa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "locator": { "kind": "table", "value": "Page 4, downstream benchmark table and named row" }, "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p_triviaqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "release_id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "review_status": "verified", "source_id": "source_google_deepmind_support_vaultgemma_report", "source_revision_id": "revision_google_deepmind_revision_support_vaultgemma_report", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_androidworld", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Pixel emulator pool" }, "prompting": { "status": "disclosed", "value": "Mobile-adapted system instruction" }, "sample_selection": { "status": "disclosed", "value": "Benchmark task set adapted for mobile" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Mobile action space with unsupported browser functions excluded" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_androidworld", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "locator": { "kind": "section", "value": "Pages 2 through 5, named benchmark methodology subsection" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_computer_use", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_androidworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "release_id": "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "review_status": "verified", "source_id": "source_google_deepmind_support_computer_use_eval_info", "source_revision_id": "revision_google_deepmind_revision_support_computer_use_eval_info", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_online_mind2web", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Anchor environment and actuation; default API sampling settings" }, "prompting": { "status": "disclosed", "value": "Benchmark system instruction with search and direct navigation functions excluded" }, "sample_selection": { "status": "disclosed", "value": "Three independent human judgments per task with majority decision" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Computer-use action tools enabled with search and direct navigation excluded" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_online_mind2web", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "locator": { "kind": "section", "value": "Pages 2 through 5, named benchmark methodology subsection" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_computer_use", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_online_mind2web" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "release_id": "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "review_status": "verified", "source_id": "source_google_deepmind_support_computer_use_eval_info", "source_revision_id": "revision_google_deepmind_revision_support_computer_use_eval_info", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_webvoyager", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Browserbase normalized harness" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Human evaluation on a common query set" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Computer-use action tools enabled" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_webvoyager", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "locator": { "kind": "section", "value": "Pages 2 through 5, named benchmark methodology subsection" }, "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_computer_use", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model_webvoyager" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "release_id": "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "review_status": "verified", "source_id": "source_google_deepmind_support_computer_use_eval_info", "source_revision_id": "revision_google_deepmind_revision_support_computer_use_eval_info", "source_type": "technical_report", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Code execution enabled" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Tools disabled" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_humanitys_last_exam", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_lmarena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_lmarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_matharena_apex", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_matharena_apex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_matharena_apex" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_simpleqa_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_simpleqa_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_simpleqa_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Terminal tool available" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_terminal_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_terminal_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_vending_bench_2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_vending_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_vending_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_video_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_video_mmmu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_video_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_webdev_arena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "State-of-the-art reasoning, Build anything, and Plan anything sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3_webdev_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "release_id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini 3 Flash: frontier intelligence at scale and For developers sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Tools disabled" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_humanitys_last_exam", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini 3 Flash: frontier intelligence at scale and For developers sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_lmarena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_lmarena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini 3 Flash: frontier intelligence at scale and For developers sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_lmarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini 3 Flash: frontier intelligence at scale and For developers sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_swe_bench_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini 3 Flash: frontier intelligence at scale and For developers sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "release_id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "heading", "value": "Elevating reasoning and Navigating complex scientific domains sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_cmt_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_cmt_benchmark", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "heading", "value": "Elevating reasoning and Navigating complex scientific domains sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_cmt_benchmark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_codeforces", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "heading", "value": "Elevating reasoning and Navigating complex scientific domains sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Tools disabled" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_humanitys_last_exam", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "heading", "value": "Elevating reasoning and Navigating complex scientific domains sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_international_chemistry_olympiad_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_icho_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "heading", "value": "Elevating reasoning and Navigating complex scientific domains sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_icho_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_imo_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_imo_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "heading", "value": "Elevating reasoning and Navigating complex scientific domains sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_imo_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_international_physics_olympiad_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_ipho_2025", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "locator": { "kind": "heading", "value": "Elevating reasoning and Navigating complex scientific domains sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e_ipho_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_arc_agi_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Release introduction, benchmark paragraph" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "release_id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arena_ai_text_leaderboard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_arena_ai_text", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Cost-efficiency without compromise, benchmark paragraph" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_arena_ai_text" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Cost-efficiency without compromise, benchmark paragraph" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Cost-efficiency without compromise, benchmark paragraph" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "release_id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_aime_2026", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Tools disabled" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_aime_2026", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Benchmark Results table, named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_aime_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_4_model_card", "source_revision_id": "revision_google_deepmind_revision_support_gemma_4_model_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_big_bench_extra_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_bigbench_extra_hard", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Benchmark Results table, named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_bigbench_extra_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_4_model_card", "source_revision_id": "revision_google_deepmind_revision_support_gemma_4_model_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_codeforces", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Benchmark Results table, named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_4_model_card", "source_revision_id": "revision_google_deepmind_revision_support_gemma_4_model_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_gpqa_diamond", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Benchmark Results table, named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_4_model_card", "source_revision_id": "revision_google_deepmind_revision_support_gemma_4_model_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Search enabled" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_humanitys_last_exam", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Benchmark Results table, named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_4_model_card", "source_revision_id": "revision_google_deepmind_revision_support_gemma_4_model_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_livecodebench_v6", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Benchmark Results table, named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_4_model_card", "source_revision_id": "revision_google_deepmind_revision_support_gemma_4_model_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmlu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Benchmark Results table, named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_4_model_card", "source_revision_id": "revision_google_deepmind_revision_support_gemma_4_model_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmmlu", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Benchmark Results table, named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_4_model_card", "source_revision_id": "revision_google_deepmind_revision_support_gemma_4_model_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmmu_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "locator": { "kind": "table", "value": "Benchmark Results table, named row" }, "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "review_status": "verified", "source_id": "source_google_deepmind_support_gemma_4_model_card", "source_revision_id": "revision_google_deepmind_revision_support_gemma_4_model_card", "source_type": "model_card", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_charxiv_reasoning", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini Flash: frontier performance for agents and coding, first paragraph" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gdpval_aa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_gdpval_aa", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini Flash: frontier performance for agents and coding, first paragraph" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_gdpval_aa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Agentic tool-use harness disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_mcp_atlas", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini Flash: frontier performance for agents and coding, first paragraph" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Agentic terminal harness disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_terminal_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Gemini Flash: frontier performance for agents and coding, first paragraph" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action_terminal_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "release_id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_big_sleep_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Google Big Sleep evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_big_sleep_evaluation", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_cyber_text_coding", "locator": { "kind": "heading", "value": "Flash Cyber benchmark results section" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_cyber", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_big_sleep_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_chrome_production_commit_scanning_pipeline", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Google Chrome production commit scanning pipeline" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Previously undisclosed vulnerabilities" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_chrome_commit_scan", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_cyber_text_coding", "locator": { "kind": "heading", "value": "Flash Cyber benchmark results section" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_cyber", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_chrome_commit_scan" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CodeMender agent configured for repeated model calls" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_cybergym", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_cyber_text_coding", "locator": { "kind": "heading", "value": "Flash Cyber benchmark results section" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_cyber", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CodeMender multi-agent orchestration" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_cybergym", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Flash model quality, Flash-Lite, and Flash Cyber sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_deepswe", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_deepswe", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Flash model quality, Flash-Lite, and Flash Cyber sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_deepswe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gdm_mrcr_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_gdm_mrcr_v2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Flash model quality, Flash-Lite, and Flash Cyber sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_gdm_mrcr_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gdpval_aa_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_gdpval_aa_v2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Flash model quality, Flash-Lite, and Flash Cyber sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_gdpval_aa_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_mle_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_mle_bench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Flash model quality, Flash-Lite, and Flash Cyber sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_mle_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Computer-use tools disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_osworld_verified", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Flash model quality, Flash-Lite, and Flash Cyber sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_swe_bench_pro", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Flash model quality, Flash-Lite, and Flash Cyber sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Agentic terminal harness disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_terminal_bench_2", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Flash model quality, Flash-Lite, and Flash Cyber sections" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5_terminal_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "release_id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_automation_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_automationbench", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Better intelligence for complex workflows, benchmark paragraphs" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_automationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_deepswe_v1_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_deepswe_v1_1", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Better intelligence for complex workflows, benchmark paragraphs" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_deepswe_v1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_frontiercode_v2_main", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_frontiercode_v2_main", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Better intelligence for complex workflows, benchmark paragraphs" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_frontiercode_v2_main" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_gdp_pdf", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_gdp_pdf", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Better intelligence for complex workflows, benchmark paragraphs" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_gdp_pdf" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_webdev_arena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_webdev_arena", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "locator": { "kind": "heading", "value": "Better intelligence for complex workflows, benchmark paragraphs" }, "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "provenance": { "record_ids": [ "occurrence_google_deepmind_occurrence_google_deepmind_20260813_introducing_gemini_3_7_flash_webdev_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "release_id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "review_status": "verified", "source_id": "source_google_deepmind_source_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_revision_id": "revision_google_deepmind_revision_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "source_type": "launch_page", "summary": "The source names this benchmark in public reporting for the released Google DeepMind model." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific few-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_arc_challenge", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_easy", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_arc_easy", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_arc_easy" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_boolq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_boolq", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_boolq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ms_coco_captions", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_coco_captions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_coco_captions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_flickr30k_captions", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_flickr30k_captions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_flickr30k_captions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "eight-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific four-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "five-shot or model-variant-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_openbookqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_openbookqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_openbookqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_piqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_social_iqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_social_iqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_social_iqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_vqav2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_vqav2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_vqav2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "in-house evaluation platform and published task tooling" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "five-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_chameleon_winogrande", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "locator": { "kind": "table", "value": "Tables 6 and 7" }, "model_id": "model_meta_model_meta_chameleon", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_chameleon_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "release_id": "release_meta_derived_20240618_meta_chameleon", "review_status": "verified", "source_id": "source_meta_paper_chameleon", "source_revision_id": "revision_meta_revision_paper_chameleon", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific few-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_arc_challenge", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_easy", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_arc_easy", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_arc_easy" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cute", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_cute", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_cute" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_flores_101", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_flores_101", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_flores_101" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hellaswag_noise", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_hellaswag_noise", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_hellaswag_noise" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot or source-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific three-shot or model-card configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "five-shot or model-variant-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_phonology_bench_g2p", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_phonology_bench_g2p", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_phonology_bench_g2p" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_piqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_dynamic_blt_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "locator": { "kind": "table", "value": "Tables 1, 3, 4, and 5" }, "model_id": "model_meta_model_meta_dynamic_blt", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_dynamic_blt_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "release_id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "review_status": "verified", "source_id": "source_meta_paper_blt", "source_revision_id": "revision_meta_revision_paper_blt", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_c4", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_c4", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_c4" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cnn_dailymail", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_cnn_dailymail", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_cnn_dailymail" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cosmopedia_coherence", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_cosmopedia_coherence", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_cosmopedia_coherence" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_flores", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_flores", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_flores" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_project_gutenberg", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_gutenberg", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_gutenberg" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_lcfo_short_summary_condition", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_lcfo_five", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_lcfo_five" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_lcfo_medium_summary_condition", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_lcfo_ten", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_lcfo_ten" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_lcfo_long_summary_condition", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_lcfo_twenty", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_lcfo_twenty" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_rocstories", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_rocstories", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_rocstories" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_wikipedia_english", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_wikipedia_en", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_wikipedia_en" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_xsum", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_large_concept_models_xsum", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "locator": { "kind": "table", "value": "Tables 3, 7, and 10 through 13" }, "model_id": "model_meta_model_meta_large_concept_models", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_large_concept_models_xsum" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "release_id": "release_meta_derived_20241212_meta_large_concept_models", "review_status": "verified", "source_id": "source_meta_paper_lcm", "source_revision_id": "revision_meta_revision_paper_lcm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper code-generation evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot or source-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_code_llama_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_code", "locator": { "kind": "table", "value": "Figure 9 and Table 5 code-checkpoint evaluation" }, "model_id": "model_meta_model_meta_layerskip_code_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_code_llama_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_code_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper code-generation evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific three-shot or model-card configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_code_llama_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_code", "locator": { "kind": "table", "value": "Figure 9 and Table 5 code-checkpoint evaluation" }, "model_id": "model_meta_model_meta_layerskip_code_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_code_llama_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_code_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific few-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_arc_challenge", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_easy", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_arc_easy", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_arc_easy" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_boolq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_boolq", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_boolq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_commonsenseqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "seven-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_commonsenseqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_commonsenseqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "eight-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot or source-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific four-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific three-shot or model-card configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "five-shot or model-variant-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_natural_questions", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_natural_questions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_natural_questions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_openbookqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_openbookqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_openbookqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_piqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_race_high", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_race_high", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_race_high" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_race_middle", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_race_middle", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_race_middle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_triviaqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_triviaqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_triviaqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "EleutherAI language-model evaluation harness" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "five-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_layerskip_llama_winogrande", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "locator": { "kind": "table", "value": "Tables 1 and 2 continual-pretraining evaluation" }, "model_id": "model_meta_model_meta_layerskip_llama", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_layerskip_llama_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "review_status": "verified", "source_id": "source_meta_paper_layerskip", "source_revision_id": "revision_meta_revision_paper_layerskip", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_adversarial_squad", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_adversarial_squad", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_adversarial_squad" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_agieval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_agieval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_agieval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_art_history", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_art_history", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_art_history" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_biology", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_biology", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_biology" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_calculus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_calculus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_calculus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_chemistry", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_chemistry", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_chemistry" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_english_language", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_english_language", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_english_language" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_english_literature", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_english_literature", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_english_literature" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_environmental_science", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_environmental_science", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_environmental_science" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_macroeconomics", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_macroeconomics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_macroeconomics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_microeconomics", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_microeconomics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_microeconomics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_physics", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_physics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_physics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_psychology", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_psychology", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_psychology" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_statistics", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_statistics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_statistics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_united_states_government", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_us_government", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_us_government" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_united_states_history", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_us_history", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_us_history" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ap_world_history", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ap_world_history", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ap_world_history" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_api_bank", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_api_bank", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_api_bank" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gorilla_api_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_api_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_api_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific few-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_arc_challenge", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_bfcl", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_bfcl" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "three-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_big_bench_hard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_big_bench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_commonsenseqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "seven-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_commonsenseqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_commonsenseqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyberseceval_code_interpreter_abuse", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberSecEval and report-specific testing frameworks" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_code_interpreter_abuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Cybersecurity and weapons-safety subsection" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_code_interpreter_abuse" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyberseceval_insecure_coding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberSecEval and report-specific testing frameworks" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_insecure_coding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Cybersecurity and weapons-safety subsection" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_insecure_coding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyberseceval_text_prompt_injection", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberSecEval and report-specific testing frameworks" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_text_prompt_injection", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Cybersecurity and weapons-safety subsection" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_text_prompt_injection" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyberseceval_vulnerability_identification", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberSecEval and report-specific testing frameworks" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_vulnerability_identification", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Cybersecurity and weapons-safety subsection" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_cyberseceval_vulnerability_identification" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "three randomly selected few-shot examples" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_drop", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_dynabench_squad", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_dynabench_squad", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_dynabench_squad" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gmat_quantitative", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_gmat_quantitative", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gmat_quantitative" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gmat_verbal", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_gmat_verbal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gmat_verbal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_gpqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gre_physics", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_gre_physics", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gre_physics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gre_general_test_quantitative_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_gre_quantitative", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gre_quantitative" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gre_general_test_verbal_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_gre_verbal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gre_verbal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "eight-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gsm_plus", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_gsm_plus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_gsm_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot or source-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_humaneval_plus", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_humaneval_plus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_humaneval_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "default benchmark configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_ifeval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_infinitebench_en_mc", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_infinitebench_en_mc", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_infinitebench_en_mc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_infinitebench_en_qa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_infinitebench_en_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_infinitebench_en_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_law_school_admission_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_lsat", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_lsat" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_many_shot_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_many_shot_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_many_shot_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific four-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific three-shot or model-card configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mbpp_evalplus_base", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_mbpp_evalplus_base", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_mbpp_evalplus_base" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_meta_attack_automation_framework", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberSecEval and report-specific testing frameworks" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_meta_attack_automation", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Cybersecurity and weapons-safety subsection" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_meta_attack_automation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_meta_spear_phishing_benchmark", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "CyberSecEval and report-specific testing frameworks" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_meta_spear_phishing", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Cybersecurity and weapons-safety subsection" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_meta_spear_phishing" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_mgsm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "five-shot or model-variant-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_mmlu_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "five-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_multilingual_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_multilingual_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_multiple", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_multiple" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_nexus", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "open-source prompt configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_nexus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_nexus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_nih_multi_needle", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_nih_multi_needle", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_nih_multi_needle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_openbookqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_openbookqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_openbookqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_paws", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_paws", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_paws" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_piqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_qasper", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_qasper", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_qasper" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_quac", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "one-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_quac", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_quac" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_quality", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_quality", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_quality" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_race", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_race", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_race" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_sat_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_sat_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_sat_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_sat_reading", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "few-shot where multiple official exam sets were available" }, "sample_selection": { "status": "disclosed", "value": "official exam sets with image-dependent questions excluded" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_sat_reading", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 17 proficiency exam rows" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_sat_reading" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_social_iqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_social_iqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_social_iqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_squad_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "one-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_squad_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_squad_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_squality", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_squality", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 16 and Tables 18 through 22" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_squality" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "five-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_winogrande", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_worldsense", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation pipeline" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_1_worldsense", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "table", "value": "Table 8 pre-training benchmark inventory" }, "model_id": "model_meta_model_meta_llama_3_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_1_worldsense" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "release_id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "review_status": "verified", "source_id": "source_meta_paper_llama_3_herd", "source_revision_id": "revision_meta_revision_paper_llama_3_herd", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_agieval_english", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "default few-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_agieval_english", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_agieval_english" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ai2d", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_ai2d", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "image reasoning benchmark tables" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_ai2d" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_vision_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_vision_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific few-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_arc_challenge", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_bfcl_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_bfcl_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_chartqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_chartqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "image reasoning benchmark tables" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_vision_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_vision_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_docvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "image reasoning benchmark tables" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_vision_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_vision_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "three randomly selected few-shot examples" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_drop", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_gpqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "eight-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "default benchmark configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_ifeval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_infinitebench_en_mc", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_infinitebench_en_mc", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_infinitebench_en_mc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_infinitebench_en_qa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_infinitebench_en_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_infinitebench_en_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_infographicsvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_infographicsvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "image reasoning benchmark tables" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_infographicsvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_vision_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_vision_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific four-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_mathvista", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "image reasoning benchmark tables" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_vision_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_vision_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_mgsm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "five-shot or model-variant-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_mmmu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "image reasoning benchmark tables" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_vision_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_vision_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmmu_pro_standard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_mmmu_pro_standard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "image reasoning benchmark tables" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mmmu_pro_standard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_vision_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_vision_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmmu_pro_vision", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_mmmu_pro_vision", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "image reasoning benchmark tables" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_mmmu_pro_vision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_vision_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_vision_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "five-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_multilingual_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_multilingual_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_needle_in_a_haystack", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_needle_in_haystack", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_needle_in_haystack" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_nexus", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "open-source prompt configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_nexus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_nexus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_nih_multi_needle", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_nih_multi_needle", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_nih_multi_needle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_open_rewrite", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_open_rewrite", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_open_rewrite" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_quac", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "one-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quac", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quac" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific few-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_arc_challenge", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_bfcl_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_bfcl_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_gpqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "eight-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "configuration stated in the benchmark table" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "default benchmark configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_ifeval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific four-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_mgsm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "five-shot or model-variant-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_nexus", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "open-source prompt configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_nexus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_nexus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_open_rewrite", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_open_rewrite", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_open_rewrite" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_tldr9", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "one-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_quantized_tldr9_plus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "locator": { "kind": "repository_path", "value": "instruction-tuned benchmark table quantized columns" }, "model_id": "model_meta_model_meta_llama_3_2_quantized", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_quantized_tldr9_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "release_id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_text_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_text_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_squad_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "one-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_squad_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_squad_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_textvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_textvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "image reasoning benchmark tables" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_textvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_vision_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_vision_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_tldr9", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "one-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_tldr9_plus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "repository heading for each text benchmark" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_tldr9_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_2_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_vqav2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal evaluations library" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_2_vqav2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "image reasoning benchmark tables" }, "model_id": "model_meta_model_meta_llama_3_2", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_2_vqav2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "release_id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "review_status": "verified", "source_id": "source_meta_tech_llama_3_2_vision_card", "source_revision_id": "revision_meta_revision_tech_llama_3_2_vision_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_agieval_english", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "default few-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_agieval_english", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_agieval_english" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific few-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_arc_challenge", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "three-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_big_bench_hard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_big_bench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_boolq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_boolq", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_boolq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_commonsenseqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "seven-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_commonsenseqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_commonsenseqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_drop", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "three randomly selected few-shot examples" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_drop", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_drop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gpqa_main", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_gpqa_main", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_gpqa_main" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "eight-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot or source-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific four-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "five-shot or model-variant-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_quac", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "one-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_quac", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_quac" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_squad_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "one-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_squad_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_squad_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_triviaqa_wiki", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "five-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_triviaqa_wiki", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_triviaqa_wiki" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official repository evaluation configuration" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompt described under the repository heading" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific split described under the repository heading" }, "shot_configuration": { "status": "disclosed", "value": "five-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_3_winogrande", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "locator": { "kind": "repository_path", "value": "repository headings for each benchmark" }, "model_id": "model_meta_model_meta_llama_3", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_3_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "release_id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "review_status": "verified", "source_id": "source_meta_tech_llama_3_eval", "source_revision_id": "revision_meta_revision_tech_llama_3_eval", "source_type": "official_repository", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_chartqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_chartqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_docvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_gpqa_diamond", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_livecodebench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific four-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_math", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_mathvista", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific three-shot or model-card configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_mgsm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "five-shot or model-variant-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "configuration stated in the benchmark table" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_mmlu_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_mmmu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "configuration stated in the benchmark table" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_mmmu_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mtob_full_book", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "configuration stated in the benchmark table" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_mtob_full_book", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mtob_full_book" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mtob_half_book", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "configuration stated in the benchmark table" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_mtob_half_book", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_mtob_half_book" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_tydiqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta model-card evaluation" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "split stated in the benchmark table when provided" }, "shot_configuration": { "status": "disclosed", "value": "one-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_llama_4_tydiqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "locator": { "kind": "repository_path", "value": "pre-trained and instruction-tuned benchmark tables" }, "model_id": "model_meta_model_meta_llama_4", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_llama_4_tydiqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "release_id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "review_status": "verified", "source_id": "source_meta_tech_llama_4_card", "source_revision_id": "revision_meta_revision_tech_llama_4_card", "source_type": "model_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_apps_intro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_apps_intro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_apps_intro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific few-shot or zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_arc_challenge", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_arc_challenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_bigpatent", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_bigpatent", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_bigpatent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cnn_dailymail", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_cnn_dailymail", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_cnn_dailymail" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_codecontests", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_codecontests", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_codecontests" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_copa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_copa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_copa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "eight-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_gsm8k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_hellaswag", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot or source-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_humaneval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific three-shot or model-card configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_mbpp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "five-shot or model-variant-specific configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_mmlu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_multi_news", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_multi_news", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_multi_news" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_natural_questions", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_natural_questions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_natural_questions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_orangesum", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_orangesum", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_orangesum" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_piqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_piqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_pn_summary", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_pn_summary", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_pn_summary" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_samsum", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_samsum", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_samsum" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_social_iqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_social_iqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_social_iqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_thaisum", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_thaisum", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_thaisum" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_triviaqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_triviaqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_triviaqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_wikisummary", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_wikisummary", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_wikisummary" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_xsum", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "paper-specific automatic evaluation" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "named benchmark split or task set described at the cited locator" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_multi_token_prediction_xsum", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "locator": { "kind": "table", "value": "Tables 1 and S8 plus Figures 4, 5, and S12" }, "model_id": "model_meta_model_meta_multi_token_prediction", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_multi_token_prediction_xsum" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "release_id": "release_meta_derived_20240618_multi_token_prediction", "review_status": "verified", "source_id": "source_meta_paper_mtp", "source_revision_id": "revision_meta_revision_paper_mtp", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_abc_bench_fragment_design", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_fragment", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_fragment" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_abc_bench_liquid_handling", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_liquid", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_liquid" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_abc_bench_screening_evasion", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_screening", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_abc_bench_screening" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_agentdojo", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_agentdojo", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_agentdojo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_agentharm_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_agentharm_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_agentharm_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_agentic_misalignment", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_agentic_misalignment", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_agentic_misalignment" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_airs_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_airs_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_airs_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_alignment_faking", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_alignment_faking", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_alignment_faking" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_babyvision", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_babyvision", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_babyvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_biodesign_tools", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_biodesign_tools", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_biodesign_tools" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_biotier_refusal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_biotier_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_biotier_refusal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_catastrophic_cyber_misuse", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_catastrophic_cyber_misuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_catastrophic_cyber_misuse" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_charxiv_reasoning", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_chemical_agents_refusal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_chemical_agents_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_chemical_agents_refusal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_curated_ctfs", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_curated_ctfs", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_curated_ctfs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cybench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyber_misuse_chat", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cyber_misuse_chat", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cyber_misuse_chat" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyberbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cybergym", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyberseceval_code_interpreter_abuse", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberseceval_code_interpreter_abuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberseceval_code_interpreter_abuse" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyberseceval_insecure_coding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberseceval_insecure_coding", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cyberseceval_insecure_coding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyscenariobench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_cyscenariobench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_cyscenariobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_deceptionbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_deceptionbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_deceptionbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_deepsearchqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_deepsearchqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_deepsearchqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_deepswe_v1_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_deepswe_v1_1", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_deepswe_v1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_exploitgym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_exploitgym", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_exploitgym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_finance_agent_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_finance_agent_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_finance_agent_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_fortress", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_fortress", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_fortress" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gdm_situational_awareness", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_gdm_situational_awareness", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_gdm_situational_awareness" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gdm_stealth", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_gdm_stealth", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_gdm_stealth" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gdpval_aa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_gdpval_aa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_gdpval_aa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_grayswan_art", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_grayswan_art", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_grayswan_art" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_healthbench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_healthbench_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_healthbench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hle_calibration", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_hle_calibration", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_hle_calibration" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hpct", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_hpct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_hpct" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_humanitys_last_exam", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_internal_sycophancy", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_internal_sycophancy", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_internal_sycophancy" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_jobbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_jobbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_jobbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mask", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_mask", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_mask" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mbct", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_mbct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_mbct" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_mcp_atlas", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mrcr_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_mrcr_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_mrcr_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_or_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_or_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_or_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_osworld_2_0", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_osworld_2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_osworld_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_osworld_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_poly_guard_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_poly_guard_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_poly_guard_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_protocolqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_protocolqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_redcode", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_redcode", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_redcode" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_save_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_save_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_save_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_lab_bench_seqqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_seqqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_seqqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_shade_arena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_shade_arena", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_shade_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_social_engineering_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_social_engineering_eval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_social_engineering_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_strongreject_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_strongreject_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_strongreject_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_bench_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_swe_bench_verified_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_bench_verified_hard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_bench_verified_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_swe_pi_agent", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_pi_agent", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_pi_agent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_swe_pi_synthetic", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_pi_synthetic", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_swe_pi_synthetic" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_terminal_bench_2_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_terminal_bench_2_1", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_terminal_bench_2_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_toolathlon_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_toolathlon_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_toolathlon_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_vct", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_vct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_vct" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_vibecodebench_v1_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_vibecodebench_v1_1", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_vibecodebench_v1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_webarena_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_webarena_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_webarena_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_widesearch", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Meta Model API and benchmark-specific harnesses" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_widesearch", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "general capabilities figure and per-benchmark details" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_widesearch" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_wmdp_bio", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_wmdp_bio", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_wmdp_bio" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_wmdp_chem", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_1_1_wmdp_chem", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2 plus named public safety subsections" }, "model_id": "model_meta_model_meta_muse_spark_1_1", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_1_1_wmdp_chem" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_meta_20260709_introducing_muse_spark_1_1", "review_status": "verified", "source_id": "source_meta_report_muse_1_1", "source_revision_id": "revision_meta_revision_report_muse_1_1", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_abc_bench_fragment_design", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_abc_bench_fragment", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_abc_bench_fragment" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_abc_bench_liquid_handling", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_abc_bench_liquid", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_abc_bench_liquid" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_agentdojo", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_agentdojo", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_agentdojo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_agentharm", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_agentharm", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_agentharm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_agentic_cybermisuse", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_agentic_cybermisuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_agentic_cybermisuse" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_agentic_misalignment", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_agentic_misalignment", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_agentic_misalignment" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_airs_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_airs_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_airs_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_alignment_faking", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_alignment_faking", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_alignment_faking" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_arc_agi_2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_biodesign_tools", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_biodesign_tools", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_biodesign_tools" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_biotier_refusal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_biotier_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_biotier_refusal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_charxiv_reasoning", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_chemical_agents_refusal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_chemical_agents_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_chemical_agents_refusal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cybench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_cybench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_cybench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyber_benign_use", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_cyber_benign_use", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_cyber_benign_use" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyber_misuse", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_cyber_misuse", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_cyber_misuse" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_cybergym", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cyscenariobench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_cyscenariobench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_cyscenariobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_deceptionbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_deceptionbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_deceptionbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_deepsearchqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_deepsearchqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_deepsearchqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_erqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_erqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_erqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_fortress", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_fortress", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_fortress" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_frontierscience_research", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_frontier_science_research", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_frontier_science_research" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gdm_stealth", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_gdm_stealth", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_gdm_stealth" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gdpval_aa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_gdpval_aa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_gdpval_aa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_gpqa_diamond", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_grayswan_art", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_grayswan_art", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_grayswan_art" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_healthbench_hard", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_healthbench_hard", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_healthbench_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hle_calibration", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_hle_calibration", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_hle_calibration" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hpct", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_hpct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_hpct" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_humanitys_last_exam", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_iheval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_iheval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_iheval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_impossiblebench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_impossiblebench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_impossiblebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_internal_sycophancy", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_internal_sycophancy", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_internal_sycophancy" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_international_physics_olympiad_2025_theory", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_ipho_2025", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_ipho_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_livecodebench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_livecodebench_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_livecodebench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mask", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_mask", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_mask" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mbct", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_mbct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_mbct" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_medxpertqa_multimodal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_medxpertqa_multimodal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_medxpertqa_multimodal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_medxpertqa_text", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_medxpertqa_text", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_medxpertqa_text" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mle_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_mle_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_mle_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_mmmu_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_or_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_or_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_or_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_prompted_evaluation_awareness", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_prompted_evaluation_awareness", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_prompted_evaluation_awareness" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_lab_bench_protocolqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_protocolqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_protocolqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_screenspot_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_secure_code_generation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_secure_code_generation", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_secure_code_generation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_lab_bench_seqqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_seqqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_seqqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_severe_cybermisuse_refusal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_severe_cybermisuse_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_severe_cybermisuse_refusal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_shade_arena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_shade_arena", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_shade_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_simpleqa_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_simpleqa_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_simpleqa_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_simplevqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_simplevqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_simplevqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_social_engineering_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_social_engineering_eval", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_social_engineering_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_social_engineering_refusal", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_social_engineering_refusal", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_social_engineering_refusal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_strongreject_v2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_strongreject_v2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_strongreject_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_swe_bench_pro", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_swe_bench_verified", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_tau_bench_telecom", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_tau_bench_telecom", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_tau_bench_telecom" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_terminal_bench_2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_terminal_bench_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_vct", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_vct", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_vct" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_wmdp_bio", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_wmdp_bio", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_wmdp_bio" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_wmdp_chem", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_wmdp_chem", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_wmdp_chem" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_wmdp_cyber", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific safety and preparedness protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_wmdp_cyber", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "section", "value": "preparedness scorecard Tables 1 and 2" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_wmdp_cyber" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_safety", "source_revision_id": "revision_meta_revision_report_muse_safety", "source_type": "system_card", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_zerobench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "report-specific evaluation protocol" }, "prompting": { "status": "disclosed", "value": "benchmark-specific prompting described in the report" }, "sample_selection": { "status": "disclosed", "value": "benchmark-specific public or report-defined split" }, "shot_configuration": { "status": "disclosed", "value": "single attempt unless a benchmark-specific exception is stated" }, "tool_use": { "status": "disclosed", "value": "benchmark-specific tool configuration described in the report" } }, "id": "occurrence_meta_occurrence_meta_muse_spark_zerobench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "locator": { "kind": "table", "value": "per-benchmark details and contemplating-mode subsections" }, "model_id": "model_meta_model_meta_muse_spark", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_muse_spark_zerobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "release_id": "release_meta_20260408_introducing_muse_spark", "review_status": "verified", "source_id": "source_meta_report_muse_methodology", "source_revision_id": "revision_meta_revision_report_muse_methodology", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_activitynet_qa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_activitynet_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_activitynet_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ai2d", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_ai2d", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_ai2d" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_blink", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_blink", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_blink" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cg_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_cg_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_cg_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_charades_sta", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_charades_sta", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_charades_sta" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_chartqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_chartqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ms_coco_captions", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_coco_captions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_coco_captions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_cv_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_cv_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_cv_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_docvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_dream_1k", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_dream_1k", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_dream_1k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_egoschema", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_egoschema", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_egoschema" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_eventhallusion", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_event_hallusion", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_event_hallusion" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_flickr30k_captions", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_flickr30k_captions", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_flickr30k_captions" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_infographicsvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_infographicsvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_infographicsvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_longvideobench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_longvideobench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_longvideobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_lvbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_lvbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_lvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_mlvu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot chain-of-thought configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_mmmu", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_motionbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_motionbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_motionbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_mvbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_mvbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_mvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_next_qa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_next_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_next_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_nocaps", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_nocaps", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_nocaps" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ocrbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_ocrbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_ocrbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_ok_vqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_ok_vqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_ok_vqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_perception_test", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_perception_test", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_perception_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_plm_videobench_fgqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_plm_videobench_fgqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_plm_videobench_fgqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_plm_videobench_rcap", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_plm_videobench_rcap", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_plm_videobench_rcap" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_plm_videobench_rdcap", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_plm_videobench_rdcap", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_plm_videobench_rdcap" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_plm_videobench_rtloc", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_plm_videobench_rtloc", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_plm_videobench_rtloc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_plm_videobench_sgqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_plm_videobench_sgqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_plm_videobench_sgqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_pope", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_pope", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_pope" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_realworldqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_refcoco", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_refcoco", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_refcoco" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_refcoco_plus", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_refcoco_plus", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_refcoco_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_refcocog", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_refcocog", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_refcocog" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_seed_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_seed_bench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_seed_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_star_video_qa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_star_video_qa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_star_video_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_tempcompass", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_tempcompass", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_tempcompass" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_temporalbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_temporalbench", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_temporalbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_textvqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_textvqa", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_textvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_tomato", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_tomato", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_tomato" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_video_mme", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_video_mme", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_video_mme" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_videohallucer", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_videohallucer", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_videohallucer" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_vizwiz", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_vizwiz", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_vizwiz" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_vqav2", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "zero-shot configuration" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_vqav2", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_vqav2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_vsr", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "official code and paper-specific judge protocols" }, "prompting": { "status": "disclosed", "value": "benchmark-specific protocol described at the cited locator" }, "sample_selection": { "status": "disclosed", "value": "validation split unless the table names another split" }, "shot_configuration": { "status": "disclosed", "value": "benchmark-specific configuration described at the cited locator" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_meta_occurrence_meta_plm_vsr", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "locator": { "kind": "table", "value": "Tables 3 through 5 and Appendix Tables 13 through 15" }, "model_id": "model_meta_model_meta_plm", "provenance": { "record_ids": [ "occurrence_meta_occurrence_meta_plm_vsr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "release_id": "release_meta_derived_20250417_meta_perception_language_model", "review_status": "verified", "source_id": "source_meta_paper_plm", "source_revision_id": "revision_meta_revision_paper_plm", "source_type": "technical_report", "summary": "The first-party source identifies this benchmark in the released model evaluation." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_humaneval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "locator": { "kind": "heading", "value": "GPT-4o mini has been evaluated across several key benchmarks" }, "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "review_status": "verified", "source_id": "source_openai_source_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "source_revision_id": "revision_openai_revision_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names HumanEval for the released OpenAI model identity." }, { "benchmark_id": "benchmark_lmarena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_lmsys_chatbot_arena", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "locator": { "kind": "heading", "value": "GPT-4o mini has been evaluated across several key benchmarks" }, "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_lmsys_chatbot_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "review_status": "verified", "source_id": "source_openai_source_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "source_revision_id": "revision_openai_revision_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names LMSYS Chatbot Arena for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mgsm", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "locator": { "kind": "heading", "value": "GPT-4o mini has been evaluated across several key benchmarks" }, "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "review_status": "verified", "source_id": "source_openai_source_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "source_revision_id": "revision_openai_revision_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MGSM for the released OpenAI model identity." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "locator": { "kind": "heading", "value": "GPT-4o mini has been evaluated across several key benchmarks" }, "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "review_status": "verified", "source_id": "source_openai_source_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "source_revision_id": "revision_openai_revision_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "locator": { "kind": "heading", "value": "GPT-4o mini has been evaluated across several key benchmarks" }, "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "release_id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "review_status": "verified", "source_id": "source_openai_source_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "source_revision_id": "revision_openai_revision_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_year_not_specified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_aime_year_not_specified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Model performance" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_aime_year_not_specified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_openai_20240912_introducing_openai_o1_preview", "source_revision_id": "revision_openai_revision_openai_20240912_introducing_openai_o1_preview", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME (year not specified) for the released OpenAI model identity." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_bbq", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Fairness and bias evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_bbq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names BBQ for the released OpenAI model identity." }, { "benchmark_id": "benchmark_biolp_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_biolp_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "BioLP-Bench" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_biolp_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names BioLP-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_challenging_refusal_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_challenging_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_challenging_refusal_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Challenging Refusal Evaluation for the released OpenAI model identity." }, { "benchmark_id": "benchmark_changemyview", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_changemyview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "ChangeMyView evaluation" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_changemyview" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names ChangeMyView for the released OpenAI model identity." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Model performance" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_openai_20240912_introducing_openai_o1_preview", "source_revision_id": "revision_openai_revision_openai_20240912_introducing_openai_o1_preview", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Codeforces for the released OpenAI model identity." }, { "benchmark_id": "benchmark_collegiate_ctfs", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_collegiate_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Evaluation of Vulnerability Identification and Exploitation Capabilities" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_collegiate_ctfs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Collegiate CTFs for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_biology", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_gpqa_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Chemical and Biological Threat Creation Evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_gpqa_biology" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names GPQA Biology for the released OpenAI model identity." }, { "benchmark_id": "benchmark_high_school_ctfs", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_high_school_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Evaluation of Vulnerability Identification and Exploitation Capabilities" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_high_school_ctfs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names High School CTFs for the released OpenAI model identity." }, { "benchmark_id": "benchmark_human_sourced_jailbreaks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_human_sourced_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_human_sourced_jailbreaks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Human-Sourced Jailbreaks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_instruction_hierarchy_conflicts", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_instruction_hierarchy_conflicts", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreaks through custom developer messages" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_instruction_hierarchy_conflicts" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Instruction Hierarchy Conflicts for the released OpenAI model identity." }, { "benchmark_id": "benchmark_jailbreak_augmented_examples", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_jailbreak_augmented_examples", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_jailbreak_augmented_examples" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Jailbreak Augmented Examples for the released OpenAI model identity." }, { "benchmark_id": "benchmark_makemepay", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_makemepay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "MakeMePay evaluation" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_makemepay" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MakeMePay for the released OpenAI model identity." }, { "benchmark_id": "benchmark_makemesay", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_makemesay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "MakeMeSay evaluation" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_makemesay" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MakeMeSay for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mle_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_mle_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "MLE-bench" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_mle_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MLE-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Multilingual performance" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MMMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_research_engineer_interview", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_openai_research_engineer_interview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "OpenAI research engineer interviews" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_openai_research_engineer_interview" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names OpenAI Research Engineer Interview for the released OpenAI model identity." }, { "benchmark_id": "benchmark_personqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_personqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Hallucination evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_personqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names PersonQA for the released OpenAI model identity." }, { "benchmark_id": "benchmark_phrase_and_password_protection", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_phrase_and_password_protection", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreaks through custom developer messages" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_phrase_and_password_protection" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Phrase and Password Protection for the released OpenAI model identity." }, { "benchmark_id": "benchmark_production_jailbreaks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_production_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_production_jailbreaks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Production Jailbreaks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_professional_ctfs", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_professional_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Evaluation of Vulnerability Identification and Exploitation Capabilities" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_professional_ctfs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Professional CTFs for the released OpenAI model identity." }, { "benchmark_id": "benchmark_protocolqa_open_ended", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_protocolqa_open_ended", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "ProtocolQA Open-Ended" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_protocolqa_open_ended" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names ProtocolQA Open-Ended for the released OpenAI model identity." }, { "benchmark_id": "benchmark_quantbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_quantbench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Additional contextual eval: QuantBench" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_quantbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names QuantBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_simpleqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Hallucination evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names SimpleQA for the released OpenAI model identity." }, { "benchmark_id": "benchmark_standard_refusal_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_standard_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_standard_refusal_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Standard Refusal Evaluation for the released OpenAI model identity." }, { "benchmark_id": "benchmark_strongreject", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_strongreject", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_strongreject" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names StrongREJECT for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "SWE-bench Verified" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_tutor_jailbreaks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_tutor_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreaks through custom developer messages" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_tutor_jailbreaks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Tutor Jailbreaks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_wildchat", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wildchat", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wildchat" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names WildChat for the released OpenAI model identity." }, { "benchmark_id": "benchmark_wmdp_biology", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wmdp_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Chemical and Biological Threat Creation Evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wmdp_biology" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names WMDP Biology for the released OpenAI model identity." }, { "benchmark_id": "benchmark_wmdp_chemistry", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wmdp_chemistry", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Chemical and Biological Threat Creation Evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_wmdp_chemistry" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names WMDP Chemistry for the released OpenAI model identity." }, { "benchmark_id": "benchmark_xstest", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_xstest", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_introducing_openai_o1_preview_xstest" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_introducing_openai_o1_preview", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names XSTest for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_year_not_specified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_aime_year_not_specified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Performance" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_aime_year_not_specified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20240912_openai_o1_mini", "source_revision_id": "revision_openai_revision_openai_20240912_openai_o1_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME (year not specified) for the released OpenAI model identity." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_bbq", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Fairness and bias evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_bbq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names BBQ for the released OpenAI model identity." }, { "benchmark_id": "benchmark_biolp_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_biolp_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "BioLP-Bench" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_biolp_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names BioLP-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_challenging_refusal_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_challenging_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_challenging_refusal_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Challenging Refusal Evaluation for the released OpenAI model identity." }, { "benchmark_id": "benchmark_changemyview", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_changemyview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "ChangeMyView evaluation" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_changemyview" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names ChangeMyView for the released OpenAI model identity." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Performance" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20240912_openai_o1_mini", "source_revision_id": "revision_openai_revision_openai_20240912_openai_o1_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Codeforces for the released OpenAI model identity." }, { "benchmark_id": "benchmark_collegiate_ctfs", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_collegiate_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Evaluation of Vulnerability Identification and Exploitation Capabilities" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_collegiate_ctfs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Collegiate CTFs for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_gpqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Performance" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20240912_openai_o1_mini", "source_revision_id": "revision_openai_revision_openai_20240912_openai_o1_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_biology", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_gpqa_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Chemical and Biological Threat Creation Evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_gpqa_biology" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names GPQA Biology for the released OpenAI model identity." }, { "benchmark_id": "benchmark_high_school_ctfs", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_high_school_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Evaluation of Vulnerability Identification and Exploitation Capabilities" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_high_school_ctfs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names High School CTFs for the released OpenAI model identity." }, { "benchmark_id": "benchmark_human_sourced_jailbreaks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_human_sourced_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_human_sourced_jailbreaks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Human-Sourced Jailbreaks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_humaneval", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Performance" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20240912_openai_o1_mini", "source_revision_id": "revision_openai_revision_openai_20240912_openai_o1_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names HumanEval for the released OpenAI model identity." }, { "benchmark_id": "benchmark_instruction_hierarchy_conflicts", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_instruction_hierarchy_conflicts", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreaks through custom developer messages" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_instruction_hierarchy_conflicts" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Instruction Hierarchy Conflicts for the released OpenAI model identity." }, { "benchmark_id": "benchmark_jailbreak_augmented_examples", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_jailbreak_augmented_examples", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_jailbreak_augmented_examples" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Jailbreak Augmented Examples for the released OpenAI model identity." }, { "benchmark_id": "benchmark_makemepay", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_makemepay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "MakeMePay evaluation" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_makemepay" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MakeMePay for the released OpenAI model identity." }, { "benchmark_id": "benchmark_makemesay", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_makemesay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "MakeMeSay evaluation" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_makemesay" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MakeMeSay for the released OpenAI model identity." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_math", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Performance" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20240912_openai_o1_mini", "source_revision_id": "revision_openai_revision_openai_20240912_openai_o1_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MATH for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mle_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mle_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "MLE-bench" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mle_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MLE-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mmlu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Performance" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20240912_openai_o1_mini", "source_revision_id": "revision_openai_revision_openai_20240912_openai_o1_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Multilingual performance" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MMMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_research_engineer_interview", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_openai_research_engineer_interview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "OpenAI research engineer interviews" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_openai_research_engineer_interview" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names OpenAI Research Engineer Interview for the released OpenAI model identity." }, { "benchmark_id": "benchmark_personqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_personqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Hallucination evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_personqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names PersonQA for the released OpenAI model identity." }, { "benchmark_id": "benchmark_phrase_and_password_protection", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_phrase_and_password_protection", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreaks through custom developer messages" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_phrase_and_password_protection" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Phrase and Password Protection for the released OpenAI model identity." }, { "benchmark_id": "benchmark_production_jailbreaks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_production_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_production_jailbreaks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Production Jailbreaks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_professional_ctfs", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_professional_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Evaluation of Vulnerability Identification and Exploitation Capabilities" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_professional_ctfs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Professional CTFs for the released OpenAI model identity." }, { "benchmark_id": "benchmark_protocolqa_open_ended", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_protocolqa_open_ended", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "ProtocolQA Open-Ended" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_protocolqa_open_ended" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names ProtocolQA Open-Ended for the released OpenAI model identity." }, { "benchmark_id": "benchmark_quantbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_quantbench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Additional contextual eval: QuantBench" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_quantbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names QuantBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_simpleqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Hallucination evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names SimpleQA for the released OpenAI model identity." }, { "benchmark_id": "benchmark_standard_refusal_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_standard_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_standard_refusal_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Standard Refusal Evaluation for the released OpenAI model identity." }, { "benchmark_id": "benchmark_strongreject", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_strongreject", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_strongreject" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names StrongREJECT for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "SWE-bench Verified" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_tutor_jailbreaks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_tutor_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Jailbreaks through custom developer messages" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_tutor_jailbreaks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Tutor Jailbreaks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_wildchat", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wildchat", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wildchat" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names WildChat for the released OpenAI model identity." }, { "benchmark_id": "benchmark_wmdp_biology", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wmdp_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Chemical and Biological Threat Creation Evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wmdp_biology" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names WMDP Biology for the released OpenAI model identity." }, { "benchmark_id": "benchmark_wmdp_chemistry", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wmdp_chemistry", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Chemical and Biological Threat Creation Evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_wmdp_chemistry" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names WMDP Chemistry for the released OpenAI model identity." }, { "benchmark_id": "benchmark_xstest", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_xstest", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20240912_openai_o1_mini_xstest" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20240912_openai_o1_mini", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names XSTest for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_aime_2024", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2024 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_bbq", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_bbq", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Fairness and bias evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_bbq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names BBQ for the released OpenAI model identity." }, { "benchmark_id": "benchmark_biolp_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_biolp_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "BioLP-Bench" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_biolp_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names BioLP-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_challenging_refusal_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_challenging_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_challenging_refusal_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Challenging Refusal Evaluation for the released OpenAI model identity." }, { "benchmark_id": "benchmark_changemyview", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_changemyview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "ChangeMyView evaluation" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_changemyview" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names ChangeMyView for the released OpenAI model identity." }, { "benchmark_id": "benchmark_collegiate_ctfs", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_collegiate_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Evaluation of Vulnerability Identification and Exploitation Capabilities" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_collegiate_ctfs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Collegiate CTFs for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_biology", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_gpqa_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Chemical and Biological Threat Creation Evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_gpqa_biology" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names GPQA Biology for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_high_school_ctfs", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_high_school_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Evaluation of Vulnerability Identification and Exploitation Capabilities" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_high_school_ctfs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names High School CTFs for the released OpenAI model identity." }, { "benchmark_id": "benchmark_human_sourced_jailbreaks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_human_sourced_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_human_sourced_jailbreaks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Human-Sourced Jailbreaks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_instruction_hierarchy_conflicts", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_instruction_hierarchy_conflicts", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Jailbreaks through custom developer messages" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_instruction_hierarchy_conflicts" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Instruction Hierarchy Conflicts for the released OpenAI model identity." }, { "benchmark_id": "benchmark_jailbreak_augmented_examples", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_jailbreak_augmented_examples", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_jailbreak_augmented_examples" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Jailbreak Augmented Examples for the released OpenAI model identity." }, { "benchmark_id": "benchmark_livebench_coding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_livebench_coding", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_livebench_coding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names LiveBench Coding for the released OpenAI model identity." }, { "benchmark_id": "benchmark_makemepay", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_makemepay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "MakeMePay evaluation" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_makemepay" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MakeMePay for the released OpenAI model identity." }, { "benchmark_id": "benchmark_makemesay", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_makemesay", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "MakeMeSay evaluation" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_makemesay" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MakeMeSay for the released OpenAI model identity." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_math", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MATH for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mathvista", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MathVista for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mgsm", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MGSM for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mle_bench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mle_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "MLE-bench" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mle_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MLE-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmlu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Multilingual performance" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names MMMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_research_engineer_interview", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_openai_research_engineer_interview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "OpenAI research engineer interviews" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_openai_research_engineer_interview" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names OpenAI Research Engineer Interview for the released OpenAI model identity." }, { "benchmark_id": "benchmark_personqa", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_personqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Hallucination evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_personqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names PersonQA for the released OpenAI model identity." }, { "benchmark_id": "benchmark_phrase_and_password_protection", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_phrase_and_password_protection", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Jailbreaks through custom developer messages" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_phrase_and_password_protection" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Phrase and Password Protection for the released OpenAI model identity." }, { "benchmark_id": "benchmark_production_jailbreaks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_production_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_production_jailbreaks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Production Jailbreaks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_professional_ctfs", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_professional_ctfs", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Evaluation of Vulnerability Identification and Exploitation Capabilities" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_professional_ctfs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Professional CTFs for the released OpenAI model identity." }, { "benchmark_id": "benchmark_protocolqa_open_ended", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_protocolqa_open_ended", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "ProtocolQA Open-Ended" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_protocolqa_open_ended" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names ProtocolQA Open-Ended for the released OpenAI model identity." }, { "benchmark_id": "benchmark_quantbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_quantbench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Additional contextual eval: QuantBench" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_quantbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names QuantBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_simpleqa", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SimpleQA for the released OpenAI model identity." }, { "benchmark_id": "benchmark_standard_refusal_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_standard_refusal_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_standard_refusal_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Standard Refusal Evaluation for the released OpenAI model identity." }, { "benchmark_id": "benchmark_strongreject", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_strongreject", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Jailbreak evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_strongreject" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names StrongREJECT for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_tau_bench_airline", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Original benchmark prompts without custom adjustment where the source states this" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tau_bench_airline", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tau_bench_airline" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau-bench Airline for the released OpenAI model identity." }, { "benchmark_id": "benchmark_tau_bench_retail", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Original benchmark prompts without custom adjustment where the source states this" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tau_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Benchmarks for the dated snapshot" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tau_bench_retail" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "source_revision_id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau-bench Retail for the released OpenAI model identity." }, { "benchmark_id": "benchmark_tutor_jailbreaks", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tutor_jailbreaks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Jailbreaks through custom developer messages" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_tutor_jailbreaks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Tutor Jailbreaks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_wildchat", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wildchat", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wildchat" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names WildChat for the released OpenAI model identity." }, { "benchmark_id": "benchmark_wmdp_biology", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wmdp_biology", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Chemical and Biological Threat Creation Evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wmdp_biology" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names WMDP Biology for the released OpenAI model identity." }, { "benchmark_id": "benchmark_wmdp_chemistry", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wmdp_chemistry", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Chemical and Biological Threat Creation Evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_wmdp_chemistry" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names WMDP Chemistry for the released OpenAI model identity." }, { "benchmark_id": "benchmark_xstest", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI safety or Preparedness evaluation harness described in the system card" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_xstest", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "locator": { "kind": "heading", "value": "Disallowed content evaluations" }, "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20241217_openai_o1_and_new_tools_for_developers_xstest" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "release_id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "review_status": "verified", "source_id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "source_revision_id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names XSTest for the released OpenAI model identity." }, { "benchmark_id": "benchmark_osworld", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Benchmark task environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Visual computer interaction through screenshots with keyboard and pointer actions" } }, "id": "occurrence_openai_occurrence_openai_20250123_computer_using_agent_osworld", "lab_id": "openai", "lineage_id": "lineage_openai_computer_using_agent", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250123_computer_using_agent", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250123_computer_using_agent_osworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "release_id": "release_openai_20250123_computer_using_agent", "review_status": "verified", "source_id": "source_openai_source_openai_20250123_computer_using_agent", "source_revision_id": "revision_openai_revision_openai_20250123_computer_using_agent", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OSWorld for the released OpenAI model identity." }, { "benchmark_id": "benchmark_webarena", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Benchmark task environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Visual computer interaction through screenshots with keyboard and pointer actions" } }, "id": "occurrence_openai_occurrence_openai_20250123_computer_using_agent_webarena", "lab_id": "openai", "lineage_id": "lineage_openai_computer_using_agent", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250123_computer_using_agent", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250123_computer_using_agent_webarena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "release_id": "release_openai_20250123_computer_using_agent", "review_status": "verified", "source_id": "source_openai_source_openai_20250123_computer_using_agent", "source_revision_id": "revision_openai_revision_openai_20250123_computer_using_agent", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names WebArena for the released OpenAI model identity." }, { "benchmark_id": "benchmark_webvoyager", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Benchmark task environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Visual computer interaction through screenshots with keyboard and pointer actions" } }, "id": "occurrence_openai_occurrence_openai_20250123_computer_using_agent_webvoyager", "lab_id": "openai", "lineage_id": "lineage_openai_computer_using_agent", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250123_computer_using_agent", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250123_computer_using_agent_webvoyager" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "release_id": "release_openai_20250123_computer_using_agent", "review_status": "verified", "source_id": "source_openai_source_openai_20250123_computer_using_agent", "source_revision_id": "revision_openai_revision_openai_20250123_computer_using_agent", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names WebVoyager for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_aime_2024", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "locator": { "kind": "heading", "value": "Model performance" }, "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250131_openai_o3_mini", "source_revision_id": "revision_openai_revision_openai_20250131_openai_o3_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2024 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "locator": { "kind": "heading", "value": "Model performance" }, "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250131_openai_o3_mini", "source_revision_id": "revision_openai_revision_openai_20250131_openai_o3_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Codeforces for the released OpenAI model identity." }, { "benchmark_id": "benchmark_frontiermath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "locator": { "kind": "heading", "value": "Model performance" }, "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_frontiermath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250131_openai_o3_mini", "source_revision_id": "revision_openai_revision_openai_20250131_openai_o3_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names FrontierMath for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "locator": { "kind": "heading", "value": "Model performance" }, "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250131_openai_o3_mini", "source_revision_id": "revision_openai_revision_openai_20250131_openai_o3_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_livebench_coding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_livebench_coding", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "locator": { "kind": "heading", "value": "Model performance" }, "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_livebench_coding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250131_openai_o3_mini", "source_revision_id": "revision_openai_revision_openai_20250131_openai_o3_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names LiveBench Coding for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "locator": { "kind": "heading", "value": "Model performance" }, "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250131_openai_o3_mini_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "release_id": "release_openai_20250131_openai_o3_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250131_openai_o3_mini", "source_revision_id": "revision_openai_revision_openai_20250131_openai_o3_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "locator": { "kind": "heading", "value": "Model evaluation scores" }, "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250227_introducing_gpt_4_5", "source_revision_id": "revision_openai_revision_openai_20250227_introducing_gpt_4_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2025 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_gpqa", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "locator": { "kind": "heading", "value": "Model evaluation scores" }, "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250227_introducing_gpt_4_5", "source_revision_id": "revision_openai_revision_openai_20250227_introducing_gpt_4_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "locator": { "kind": "heading", "value": "Model evaluation scores" }, "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250227_introducing_gpt_4_5", "source_revision_id": "revision_openai_revision_openai_20250227_introducing_gpt_4_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "locator": { "kind": "heading", "value": "Model evaluation scores" }, "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250227_introducing_gpt_4_5", "source_revision_id": "revision_openai_revision_openai_20250227_introducing_gpt_4_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_simpleqa", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "locator": { "kind": "heading", "value": "Model evaluation scores" }, "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250227_introducing_gpt_4_5", "source_revision_id": "revision_openai_revision_openai_20250227_introducing_gpt_4_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SimpleQA for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "locator": { "kind": "heading", "value": "Model evaluation scores" }, "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250227_introducing_gpt_4_5", "source_revision_id": "revision_openai_revision_openai_20250227_introducing_gpt_4_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_lancer_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_swe_lancer_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "locator": { "kind": "heading", "value": "Model evaluation scores" }, "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250227_introducing_gpt_4_5_swe_lancer_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "release_id": "release_openai_20250227_introducing_gpt_4_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250227_introducing_gpt_4_5", "source_revision_id": "revision_openai_revision_openai_20250227_introducing_gpt_4_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Lancer Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aider_polyglot", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_aider_polyglot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Aider Polyglot for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2025 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_charxiv_descriptive", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_charxiv_descriptive", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_charxiv_descriptive" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names CharXiv Descriptive for the released OpenAI model identity." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names CharXiv Reasoning for the released OpenAI model identity." }, { "benchmark_id": "benchmark_collie", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_collie", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_collie" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names COLLIE for the released OpenAI model identity." }, { "benchmark_id": "benchmark_complexfuncbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_complexfuncbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_complexfuncbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names ComplexFuncBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_graphwalks", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Directed-graph breadth-first-search tasks distributed across long context" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_graphwalks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GraphWalks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_ifeval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names IFEval for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mathvista", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MathVista for the released OpenAI model identity." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_multi_if", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_multi_if", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_multi_if" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Multi-IF for the released OpenAI model identity." }, { "benchmark_id": "benchmark_multichallenge", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_multichallenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MultiChallenge for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_mrcr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Synthetic long conversations with repeated user requests to test coreference retrieval" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_openai_mrcr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI-MRCR for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_lancer", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_lancer", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_lancer" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Lancer for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_swe_lancer_ic_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Lancer IC Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_tau_bench_airline", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Original benchmark prompts without custom adjustment where the source states this" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_tau_bench_airline", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_tau_bench_airline" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau-bench Airline for the released OpenAI model identity." }, { "benchmark_id": "benchmark_tau_bench_retail", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Original benchmark prompts without custom adjustment where the source states this" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_tau_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_tau_bench_retail" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau-bench Retail for the released OpenAI model identity." }, { "benchmark_id": "benchmark_video_mme", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_video_mme", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "locator": { "kind": "heading", "value": "Full evaluation tables" }, "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250414_introducing_gpt_4_1_in_the_api_video_mme" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "release_id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "review_status": "verified", "source_id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "source_revision_id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Video-MME for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_aime_2024", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini", "source_revision_id": "revision_openai_revision_openai_20250416_introducing_openai_o3_and_o4_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2024 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini", "source_revision_id": "revision_openai_revision_openai_20250416_introducing_openai_o3_and_o4_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2025 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini", "source_revision_id": "revision_openai_revision_openai_20250416_introducing_openai_o3_and_o4_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names CharXiv Reasoning for the released OpenAI model identity." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini", "source_revision_id": "revision_openai_revision_openai_20250416_introducing_openai_o3_and_o4_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Codeforces for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_mathvista", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini", "source_revision_id": "revision_openai_revision_openai_20250416_introducing_openai_o3_and_o4_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MathVista for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini", "source_revision_id": "revision_openai_revision_openai_20250416_introducing_openai_o3_and_o4_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini", "source_revision_id": "revision_openai_revision_openai_20250416_introducing_openai_o3_and_o4_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_tau_bench_retail", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Original benchmark prompts without custom adjustment where the source states this" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_tau_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250416_introducing_openai_o3_and_o4_mini_tau_bench_retail" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "release_id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "review_status": "verified", "source_id": "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini", "source_revision_id": "revision_openai_revision_openai_20250416_introducing_openai_o3_and_o4_mini", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau-bench Retail for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_internal_swe_tasks", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250516_introducing_codex_openai_internal_swe_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "locator": { "kind": "heading", "value": "Coding evaluations and internal benchmarks" }, "model_id": "model_openai_model_openai_20250516_introducing_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250516_introducing_codex_openai_internal_swe_tasks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-16", "release_id": "release_openai_20250516_introducing_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20250516_introducing_codex", "source_revision_id": "revision_openai_revision_openai_20250516_introducing_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI Internal SWE Tasks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250516_introducing_codex_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "locator": { "kind": "heading", "value": "Coding evaluations and internal benchmarks" }, "model_id": "model_openai_model_openai_20250516_introducing_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250516_introducing_codex_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-16", "release_id": "release_openai_20250516_introducing_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20250516_introducing_codex", "source_revision_id": "revision_openai_revision_openai_20250516_introducing_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_aime_2024", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "review_status": "verified", "source_id": "source_openai_source_openai_20250805_introducing_gpt_oss", "source_revision_id": "revision_openai_revision_openai_20250805_introducing_gpt_oss", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2024 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "review_status": "verified", "source_id": "source_openai_source_openai_20250805_introducing_gpt_oss", "source_revision_id": "revision_openai_revision_openai_20250805_introducing_gpt_oss", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2025 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "review_status": "verified", "source_id": "source_openai_source_openai_20250805_introducing_gpt_oss", "source_revision_id": "revision_openai_revision_openai_20250805_introducing_gpt_oss", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Codeforces for the released OpenAI model identity." }, { "benchmark_id": "benchmark_healthbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Physician-informed grading procedure; later system cards disclose response-length adjustment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_healthbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "review_status": "verified", "source_id": "source_openai_source_openai_20250805_introducing_gpt_oss", "source_revision_id": "revision_openai_revision_openai_20250805_introducing_gpt_oss", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names HealthBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_humanitys_last_exam", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "review_status": "verified", "source_id": "source_openai_source_openai_20250805_introducing_gpt_oss", "source_revision_id": "revision_openai_revision_openai_20250805_introducing_gpt_oss", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Humanity's Last Exam for the released OpenAI model identity." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_mmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "review_status": "verified", "source_id": "source_openai_source_openai_20250805_introducing_gpt_oss", "source_revision_id": "revision_openai_revision_openai_20250805_introducing_gpt_oss", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_tau_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250805_introducing_gpt_oss_tau_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "release_id": "release_openai_20250805_introducing_gpt_oss", "review_status": "verified", "source_id": "source_openai_source_openai_20250805_introducing_gpt_oss", "source_revision_id": "revision_openai_revision_openai_20250805_introducing_gpt_oss", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau-bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aider_polyglot", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_aider_polyglot", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_aider_polyglot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Aider Polyglot for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2025 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names CharXiv Reasoning for the released OpenAI model identity." }, { "benchmark_id": "benchmark_factscore", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_factscore", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_factscore" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names FActScore for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_healthbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Physician-informed grading procedure; later system cards disclose response-length adjustment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_healthbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names HealthBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_longfact", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_longfact", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_longfact" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names LongFact for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI simple-evals compatible evaluation where identified by the source" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_multichallenge", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_multichallenge", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_multichallenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MultiChallenge for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250807_introducing_gpt_5_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "release_id": "release_openai_20250807_introducing_gpt_5", "review_status": "verified", "source_id": "source_openai_source_openai_20250807_introducing_gpt_5", "source_revision_id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_code_refactoring_tasks", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_code_refactoring_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Coding evaluations" }, "model_id": "model_openai_model_openai_20250915_introducing_upgrades_to_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_code_refactoring_tasks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "release_id": "release_openai_20250915_introducing_upgrades_to_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20250915_introducing_upgrades_to_codex", "source_revision_id": "revision_openai_revision_openai_20250915_introducing_upgrades_to_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI Code Refactoring Tasks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_code_review_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_code_review_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Coding evaluations" }, "model_id": "model_openai_model_openai_20250915_introducing_upgrades_to_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_code_review_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "release_id": "release_openai_20250915_introducing_upgrades_to_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20250915_introducing_upgrades_to_codex", "source_revision_id": "revision_openai_revision_openai_20250915_introducing_upgrades_to_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI Code Review Evaluation for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_frontend_preference_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_frontend_preference_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Coding evaluations" }, "model_id": "model_openai_model_openai_20250915_introducing_upgrades_to_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_openai_frontend_preference_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "release_id": "release_openai_20250915_introducing_upgrades_to_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20250915_introducing_upgrades_to_codex", "source_revision_id": "revision_openai_revision_openai_20250915_introducing_upgrades_to_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI Frontend Preference Evaluation for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Coding evaluations" }, "model_id": "model_openai_model_openai_20250915_introducing_upgrades_to_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20250915_introducing_upgrades_to_codex_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "release_id": "release_openai_20250915_introducing_upgrades_to_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20250915_introducing_upgrades_to_codex", "source_revision_id": "revision_openai_revision_openai_20250915_introducing_upgrades_to_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Adaptive reasoning" }, "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "review_status": "verified", "source_id": "source_openai_source_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "source_revision_id": "revision_openai_revision_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2025 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_codeforces", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Adaptive reasoning" }, "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "release_id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "review_status": "verified", "source_id": "source_openai_source_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "source_revision_id": "revision_openai_revision_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Codeforces for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Appendix: Model evaluations" }, "model_id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "release_id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "review_status": "verified", "source_id": "source_openai_source_openai_20251119_building_more_with_gpt_5_1_codex_max", "source_revision_id": "revision_openai_revision_openai_20251119_building_more_with_gpt_5_1_codex_max", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_lancer_ic_swe", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_swe_lancer_ic_swe", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Appendix: Model evaluations" }, "model_id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_swe_lancer_ic_swe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "release_id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "review_status": "verified", "source_id": "source_openai_source_openai_20251119_building_more_with_gpt_5_1_codex_max", "source_revision_id": "revision_openai_revision_openai_20251119_building_more_with_gpt_5_1_codex_max", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Lancer IC SWE for the released OpenAI model identity." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Appendix: Model evaluations" }, "model_id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251119_building_more_with_gpt_5_1_codex_max_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "release_id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "review_status": "verified", "source_id": "source_openai_source_openai_20251119_building_more_with_gpt_5_1_codex_max", "source_revision_id": "revision_openai_revision_openai_20251119_building_more_with_gpt_5_1_codex_max", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Terminal-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_aime_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AIME 2025 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_arc_agi_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_arc_agi_1", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_arc_agi_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names ARC-AGI-1 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names ARC-AGI-2 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names BrowseComp for the released OpenAI model identity." }, { "benchmark_id": "benchmark_browsecomp_long_context", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_browsecomp_long_context", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_browsecomp_long_context" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names BrowseComp Long Context for the released OpenAI model identity." }, { "benchmark_id": "benchmark_charxiv_reasoning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_charxiv_reasoning", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_charxiv_reasoning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names CharXiv Reasoning for the released OpenAI model identity." }, { "benchmark_id": "benchmark_frontiermath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_frontiermath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names FrontierMath for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gdpval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_gdpval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GDPval for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_graphwalks", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Directed-graph breadth-first-search tasks distributed across long context" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_graphwalks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GraphWalks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_hmmt_february_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_hmmt_february_2025", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_hmmt_february_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names HMMT February 2025 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_humanitys_last_exam", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Humanity's Last Exam for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_mmmlu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMLU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_chatgpt_factuality", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_chatgpt_factuality", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_chatgpt_factuality" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI ChatGPT Factuality for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_investment_banking_spreadsheet_tasks", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_investment_banking_spreadsheet_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_investment_banking_spreadsheet_tasks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI Investment Banking Spreadsheet Tasks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_mrcr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Synthetic long conversations with repeated user requests to test coreference retrieval" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_openai_mrcr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI-MRCR for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_scale_mcp_atlas", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_scale_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Scale MCP-Atlas for the released OpenAI model identity." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_screenspot_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names ScreenSpot Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Bench Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_bench_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-bench Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_swe_lancer_ic_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Lancer IC Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_record_2_bench_retail", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Original benchmark prompts without custom adjustment where the source states this" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_tau2_bench_retail", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_tau2_bench_retail" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau2-bench Retail for the released OpenAI model identity." }, { "benchmark_id": "benchmark_record_2_bench_telecom", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Original benchmark prompts without custom adjustment where the source states this" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_tau2_bench_telecom", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_tau2_bench_telecom" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau2-bench Telecom for the released OpenAI model identity." }, { "benchmark_id": "benchmark_toolathlon", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_toolathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Toolathlon for the released OpenAI model identity." }, { "benchmark_id": "benchmark_video_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_video_mmmu", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Detailed benchmarks" }, "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251211_introducing_gpt_5_2_video_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "release_id": "release_openai_20251211_introducing_gpt_5_2", "review_status": "verified", "source_id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "source_revision_id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Video MMMU for the released OpenAI model identity." }, { "benchmark_id": "benchmark_professional_ctf", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_professional_ctf", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Agentic coding evaluations" }, "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_professional_ctf" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20251218_introducing_gpt_5_2_codex", "source_revision_id": "revision_openai_revision_openai_20251218_introducing_gpt_5_2_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Professional CTF for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Agentic coding evaluations" }, "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20251218_introducing_gpt_5_2_codex", "source_revision_id": "revision_openai_revision_openai_20251218_introducing_gpt_5_2_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Bench Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Agentic coding evaluations" }, "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20251218_introducing_gpt_5_2_codex_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "release_id": "release_openai_20251218_introducing_gpt_5_2_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20251218_introducing_gpt_5_2_codex", "source_revision_id": "revision_openai_revision_openai_20251218_introducing_gpt_5_2_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Terminal-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_cybersecurity_ctf", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_cybersecurity_ctf", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_cybersecurity_ctf" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20260205_introducing_gpt_5_3_codex", "source_revision_id": "revision_openai_revision_openai_20260205_introducing_gpt_5_3_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Cybersecurity CTF for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gdpval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_gdpval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20260205_introducing_gpt_5_3_codex", "source_revision_id": "revision_openai_revision_openai_20260205_introducing_gpt_5_3_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GDPval for the released OpenAI model identity." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Benchmark task environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Visual computer interaction through screenshots with keyboard and pointer actions" } }, "id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_osworld_verified", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20260205_introducing_gpt_5_3_codex", "source_revision_id": "revision_openai_revision_openai_20260205_introducing_gpt_5_3_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OSWorld-Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20260205_introducing_gpt_5_3_codex", "source_revision_id": "revision_openai_revision_openai_20260205_introducing_gpt_5_3_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Bench Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_lancer_ic_diamond", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_swe_lancer_ic_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_swe_lancer_ic_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20260205_introducing_gpt_5_3_codex", "source_revision_id": "revision_openai_revision_openai_20260205_introducing_gpt_5_3_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Lancer IC Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260205_introducing_gpt_5_3_codex_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "release_id": "release_openai_20260205_introducing_gpt_5_3_codex", "review_status": "verified", "source_id": "source_openai_source_openai_20260205_introducing_gpt_5_3_codex", "source_revision_id": "revision_openai_revision_openai_20260205_introducing_gpt_5_3_codex", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Terminal-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260212_introducing_gpt_5_3_codex_spark_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "locator": { "kind": "heading", "value": "Model performance" }, "model_id": "model_openai_model_openai_20260212_introducing_gpt_5_3_codex_spark", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260212_introducing_gpt_5_3_codex_spark_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_openai_20260212_introducing_gpt_5_3_codex_spark", "review_status": "verified", "source_id": "source_openai_source_openai_20260212_introducing_gpt_5_3_codex_spark", "source_revision_id": "revision_openai_revision_openai_20260212_introducing_gpt_5_3_codex_spark", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Bench Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260212_introducing_gpt_5_3_codex_spark_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "locator": { "kind": "heading", "value": "Model performance" }, "model_id": "model_openai_model_openai_20260212_introducing_gpt_5_3_codex_spark", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260212_introducing_gpt_5_3_codex_spark_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "release_id": "release_openai_20260212_introducing_gpt_5_3_codex_spark", "review_status": "verified", "source_id": "source_openai_source_openai_20260212_introducing_gpt_5_3_codex_spark", "source_revision_id": "revision_openai_revision_openai_20260212_introducing_gpt_5_3_codex_spark", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Terminal-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_apex_agents", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_apex_agents", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_apex_agents" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names APEX-Agents for the released OpenAI model identity." }, { "benchmark_id": "benchmark_arc_agi_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_arc_agi_1", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_arc_agi_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names ARC-AGI-1 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names ARC-AGI-2 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_biglaw_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_biglaw_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_biglaw_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names BigLaw Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names BrowseComp for the released OpenAI model identity." }, { "benchmark_id": "benchmark_finance_agent", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_financeagent", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_financeagent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names FinanceAgent for the released OpenAI model identity." }, { "benchmark_id": "benchmark_frontier_science_research", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_frontier_science_research", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_frontier_science_research" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Frontier Science Research for the released OpenAI model identity." }, { "benchmark_id": "benchmark_frontiermath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_frontiermath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names FrontierMath for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gdpval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_gdpval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GDPval for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_humanitys_last_exam", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Humanity's Last Exam for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_omnidocbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_omnidocbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_omnidocbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OmniDocBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_online_mind2web", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Benchmark task environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Visual computer interaction through screenshots with keyboard and pointer actions" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_online_mind2web", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_online_mind2web" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Online-Mind2Web for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_mrcr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Synthetic long conversations with repeated user requests to test coreference retrieval" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_openai_mrcr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI-MRCR for the released OpenAI model identity." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Benchmark task environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Visual computer interaction through screenshots with keyboard and pointer actions" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_osworld_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OSWorld-Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_scale_mcp_atlas", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_scale_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Scale MCP-Atlas for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Bench Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_record_2_bench_telecom", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Original benchmark prompts without custom adjustment where the source states this" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_tau2_bench_telecom", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_tau2_bench_telecom" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau2-bench Telecom for the released OpenAI model identity." }, { "benchmark_id": "benchmark_toolathlon", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_toolathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Toolathlon for the released OpenAI model identity." }, { "benchmark_id": "benchmark_webarena_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Benchmark task environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Visual computer interaction through screenshots with keyboard and pointer actions" } }, "id": "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_webarena_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260305_introducing_gpt_5_4_webarena_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "release_id": "release_openai_20260305_introducing_gpt_5_4", "review_status": "verified", "source_id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "source_revision_id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names WebArena-Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_graphwalks", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Directed-graph breadth-first-search tasks distributed across long context" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_graphwalks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GraphWalks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_humanitys_last_exam", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Humanity's Last Exam for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_omnidocbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_omnidocbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_omnidocbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OmniDocBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_mrcr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Synthetic long conversations with repeated user requests to test coreference retrieval" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_openai_mrcr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI-MRCR for the released OpenAI model identity." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Benchmark task environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Visual computer interaction through screenshots with keyboard and pointer actions" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_osworld_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OSWorld-Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_scale_mcp_atlas", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_scale_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Scale MCP-Atlas for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Bench Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_record_2_bench_telecom", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Original benchmark prompts without custom adjustment where the source states this" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_tau2_bench_telecom", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_tau2_bench_telecom" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau2-bench Telecom for the released OpenAI model identity." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Terminal-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_toolathlon", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260317_introducing_gpt_5_4_mini_and_nano_toolathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "release_id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "review_status": "verified", "source_id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_revision_id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Toolathlon for the released OpenAI model identity." }, { "benchmark_id": "benchmark_bixbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_bixbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "locator": { "kind": "heading", "value": "Industry evaluations" }, "model_id": "model_openai_model_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_bixbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "review_status": "verified", "source_id": "source_openai_source_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "source_revision_id": "revision_openai_revision_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names BixBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_cloningqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_cloningqa", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "locator": { "kind": "heading", "value": "Industry evaluations" }, "model_id": "model_openai_model_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_cloningqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "review_status": "verified", "source_id": "source_openai_source_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "source_revision_id": "revision_openai_revision_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names CloningQA for the released OpenAI model identity." }, { "benchmark_id": "benchmark_labbench2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_labbench2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "locator": { "kind": "heading", "value": "Industry evaluations" }, "model_id": "model_openai_model_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research_labbench2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "release_id": "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "review_status": "verified", "source_id": "source_openai_source_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "source_revision_id": "revision_openai_revision_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names LABBench2 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_arc_agi_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_arc_agi_1", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_arc_agi_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names ARC-AGI-1 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_bixbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_bixbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_bixbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names BixBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names BrowseComp for the released OpenAI model identity." }, { "benchmark_id": "benchmark_capture_the_flag_challenges", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_capture_the_flag_challenges", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_capture_the_flag_challenges" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Capture-the-Flag Challenges for the released OpenAI model identity." }, { "benchmark_id": "benchmark_cot_control", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Chain-of-thought controllability evaluation suite" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_cot_control", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Capability evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_cot_control" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_https_deploymentsafety_openai_com_gpt_5_5", "source_revision_id": "revision_openai_revision_https_deploymentsafety_openai_com_gpt_5_5", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names CoT-Control for the released OpenAI model identity." }, { "benchmark_id": "benchmark_cybergym", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_cybergym", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_cybergym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names CyberGym for the released OpenAI model identity." }, { "benchmark_id": "benchmark_expert_swe", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_expert_swe", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_expert_swe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Expert-SWE for the released OpenAI model identity." }, { "benchmark_id": "benchmark_finance_agent", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_financeagent", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_financeagent" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names FinanceAgent for the released OpenAI model identity." }, { "benchmark_id": "benchmark_frontiermath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_frontiermath", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_frontiermath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names FrontierMath for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gdpval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_gdpval", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_gdpval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GDPval for the released OpenAI model identity." }, { "benchmark_id": "benchmark_genebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_genebench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_genebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GeneBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_graphwalks", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Directed-graph breadth-first-search tasks distributed across long context" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_graphwalks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GraphWalks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_healthbench", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Physician-informed grading procedure; later system cards disclose response-length adjustment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_healthbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Capability evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_healthbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_https_deploymentsafety_openai_com_gpt_5_5", "source_revision_id": "revision_openai_revision_https_deploymentsafety_openai_com_gpt_5_5", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names HealthBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_healthbench_professional", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Physician-informed grading procedure; later system cards disclose response-length adjustment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_healthbench_professional", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Capability evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_healthbench_professional" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_https_deploymentsafety_openai_com_gpt_5_5", "source_revision_id": "revision_openai_revision_https_deploymentsafety_openai_com_gpt_5_5", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names HealthBench Professional for the released OpenAI model identity." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_humanitys_last_exam", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_humanitys_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Humanity's Last Exam for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_nucleobench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_nucleobench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Capability evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_nucleobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_https_deploymentsafety_openai_com_gpt_5_5", "source_revision_id": "revision_openai_revision_https_deploymentsafety_openai_com_gpt_5_5", "source_type": "system_card", "summary": "The reviewed OpenAI reporting names Nucleobench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_officeqa_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_officeqa_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_officeqa_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OfficeQA Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_investment_banking_spreadsheet_tasks", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_openai_investment_banking_spreadsheet_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_openai_investment_banking_spreadsheet_tasks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI Investment Banking Spreadsheet Tasks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_mrcr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Synthetic long conversations with repeated user requests to test coreference retrieval" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_openai_mrcr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI-MRCR for the released OpenAI model identity." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Benchmark task environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Visual computer interaction through screenshots with keyboard and pointer actions" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_osworld_verified", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OSWorld-Verified for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Bench Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_record_2_bench_telecom", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "Original benchmark prompts without custom adjustment where the source states this" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_tau2_bench_telecom", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_tau2_bench_telecom" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Tau2-bench Telecom for the released OpenAI model identity." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Terminal-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_toolathlon", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "locator": { "kind": "heading", "value": "Evaluations" }, "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260423_introducing_gpt_5_5_toolathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "release_id": "release_openai_20260423_introducing_gpt_5_5", "review_status": "verified", "source_id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "source_revision_id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Toolathlon for the released OpenAI model identity." }, { "benchmark_id": "benchmark_genebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_genebench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_genebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "review_status": "verified", "source_id": "source_openai_source_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "source_revision_id": "revision_openai_revision_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GeneBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_labworkbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_labworkbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_labworkbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "review_status": "verified", "source_id": "source_openai_source_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "source_revision_id": "revision_openai_revision_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names LabWorkBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_lifescibench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_lifescibench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_lifescibench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "review_status": "verified", "source_id": "source_openai_source_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "source_revision_id": "revision_openai_revision_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names LifeSciBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_medchembench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_medchembench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "locator": { "kind": "heading", "value": "Model evaluations" }, "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260603_introducing_new_capabilities_to_gpt_rosalind_medchembench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "release_id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "review_status": "verified", "source_id": "source_openai_source_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "source_revision_id": "revision_openai_revision_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MedChemBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_exploitbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_exploitbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Preview evaluations" }, "model_id": "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_exploitbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "release_id": "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "review_status": "verified", "source_id": "source_openai_source_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "source_revision_id": "revision_openai_revision_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names ExploitBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_exploitgym", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_exploitgym", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Preview evaluations" }, "model_id": "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_exploitgym" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "release_id": "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "review_status": "verified", "source_id": "source_openai_source_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "source_revision_id": "revision_openai_revision_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names ExploitGym for the released OpenAI model identity." }, { "benchmark_id": "benchmark_genebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_genebench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Preview evaluations" }, "model_id": "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_genebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "release_id": "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "review_status": "verified", "source_id": "source_openai_source_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "source_revision_id": "revision_openai_revision_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GeneBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Preview evaluations" }, "model_id": "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "release_id": "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "review_status": "verified", "source_id": "source_openai_source_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "source_revision_id": "revision_openai_revision_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Terminal-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_agents_last_exam", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_agents_last_exam", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_agents_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Agents' Last Exam for the released OpenAI model identity." }, { "benchmark_id": "benchmark_arc_agi_2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_arc_agi_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_arc_agi_2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names ARC-AGI-2 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_artificial_analysis_coding_agent_index", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_artificial_analysis_coding_agent_index", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_artificial_analysis_coding_agent_index" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Artificial Analysis Coding Agent Index for the released OpenAI model identity." }, { "benchmark_id": "benchmark_artificial_analysis_intelligence_index", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_artificial_analysis_intelligence_index", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_artificial_analysis_intelligence_index" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Artificial Analysis Intelligence Index for the released OpenAI model identity." }, { "benchmark_id": "benchmark_automation_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_automationbench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_automationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names AutomationBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_benchcad", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchcad", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_benchcad" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names BenchCAD for the released OpenAI model identity." }, { "benchmark_id": "benchmark_big_finance_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_big_finance_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_big_finance_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Big Finance Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_browsecomp", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names BrowseComp for the released OpenAI model identity." }, { "benchmark_id": "benchmark_capture_the_flag_challenges", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Curated security challenge tasks" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Security task tooling and iterative execution enabled where described" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_capture_the_flag_challenges", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_capture_the_flag_challenges" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Capture-the-Flag Challenges for the released OpenAI model identity." }, { "benchmark_id": "benchmark_deepswe", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_deepswe", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_deepswe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names DeepSWE for the released OpenAI model identity." }, { "benchmark_id": "benchmark_frontiermath_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_frontiermath_v2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_frontiermath_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names FrontierMath v2 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gdpval_aa_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_gdpval_aa_v2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_gdpval_aa_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GDPval-AA v2 for the released OpenAI model identity." }, { "benchmark_id": "benchmark_genebench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_genebench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_genebench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GeneBench Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_gpqa_diamond", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GPQA Diamond for the released OpenAI model identity." }, { "benchmark_id": "benchmark_graphwalks", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Directed-graph breadth-first-search tasks distributed across long context" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_graphwalks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_graphwalks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names GraphWalks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_healthbench_professional", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Physician-informed grading procedure; later system cards disclose response-length adjustment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_healthbench_professional", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_healthbench_professional" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names HealthBench Professional for the released OpenAI model identity." }, { "benchmark_id": "benchmark_kernelgen_1p", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_kernelgen_1p", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_kernelgen_1p" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names KernelGen 1P for the released OpenAI model identity." }, { "benchmark_id": "benchmark_medchembench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_medchembench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_medchembench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MedChemBench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "The source distinguishes tool-disabled and tool-enabled configurations where both are reported" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_mmmu_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names MMMU Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_management_consulting_tasks", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_management_consulting_tasks", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_management_consulting_tasks" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI Management Consulting Tasks for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_mrcr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "Synthetic long conversations with repeated user requests to test coreference retrieval" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_mrcr", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_mrcr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI-MRCR for the released OpenAI model identity." }, { "benchmark_id": "benchmark_openai_research_debugging_evaluation", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_research_debugging_evaluation", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_openai_research_debugging_evaluation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OpenAI Research Debugging Evaluation for the released OpenAI model identity." }, { "benchmark_id": "benchmark_osworld", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Benchmark task environment" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Visual computer interaction through screenshots with keyboard and pointer actions" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_osworld", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_osworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names OSWorld for the released OpenAI model identity." }, { "benchmark_id": "benchmark_sec_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_sec_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_sec_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SEC-Bench Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "OpenAI execution infrastructure" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "The source identifies a runnable or validated benchmark subset when applicable" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_swe_bench_pro", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names SWE-Bench Pro for the released OpenAI model identity." }, { "benchmark_id": "benchmark_terminal_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_terminal_bench", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_terminal_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Terminal-Bench for the released OpenAI model identity." }, { "benchmark_id": "benchmark_toolathlon", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "Benchmark-provided tools or interfaces enabled" } }, "id": "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_toolathlon", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "locator": { "kind": "heading", "value": "Evaluation tables" }, "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "provenance": { "record_ids": [ "occurrence_openai_occurrence_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your_toolathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "release_id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "review_status": "verified", "source_id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_revision_id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "source_type": "launch_page", "summary": "The reviewed OpenAI reporting names Toolathlon for the released OpenAI model identity." }, { "benchmark_id": "benchmark_codeu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_codeu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Base Model and Tool Use and Agent; benchmark label CodeU" }, "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_codeu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240123_introducing_qwen_qwen", "source_revision_id": "revision_qwen_revision_qwen_20240123_introducing_qwen_qwen", "source_type": "launch_page", "summary": "The official Qwen release reporting names CodeU for the released Qwen model." }, { "benchmark_id": "benchmark_coursera", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_coursera", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Base Model and Tool Use and Agent; benchmark label Coursera" }, "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_coursera" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240123_introducing_qwen_qwen", "source_revision_id": "revision_qwen_revision_qwen_20240123_introducing_qwen_qwen", "source_type": "launch_page", "summary": "The official Qwen release reporting names Coursera for the released Qwen model." }, { "benchmark_id": "benchmark_gsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_gsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Base Model and Tool Use and Agent; benchmark label GSM" }, "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_gsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240123_introducing_qwen_qwen", "source_revision_id": "revision_qwen_revision_qwen_20240123_introducing_qwen_qwen", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM for the released Qwen model." }, { "benchmark_id": "benchmark_l_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_l_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Base Model and Tool Use and Agent; benchmark label L-Eval" }, "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_l_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240123_introducing_qwen_qwen", "source_revision_id": "revision_qwen_revision_qwen_20240123_introducing_qwen_qwen", "source_type": "launch_page", "summary": "The official Qwen release reporting names L-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_needle_in_a_haystack", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_needle_in_a_haystack", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Base Model and Tool Use and Agent; benchmark label Needle in a Haystack" }, "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_needle_in_a_haystack" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240123_introducing_qwen_qwen", "source_revision_id": "revision_qwen_revision_qwen_20240123_introducing_qwen_qwen", "source_type": "launch_page", "summary": "The official Qwen release reporting names Needle in a Haystack for the released Qwen model." }, { "benchmark_id": "benchmark_quality", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_quality", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Base Model and Tool Use and Agent; benchmark label QuALITY" }, "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_quality" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240123_introducing_qwen_qwen", "source_revision_id": "revision_qwen_revision_qwen_20240123_introducing_qwen_qwen", "source_type": "launch_page", "summary": "The official Qwen release reporting names QuALITY for the released Qwen model." }, { "benchmark_id": "benchmark_sfiction", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_sfiction", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Base Model and Tool Use and Agent; benchmark label SFiction" }, "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_sfiction" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240123_introducing_qwen_qwen", "source_revision_id": "revision_qwen_revision_qwen_20240123_introducing_qwen_qwen", "source_type": "launch_page", "summary": "The official Qwen release reporting names SFiction for the released Qwen model." }, { "benchmark_id": "benchmark_toefl", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_toefl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Base Model and Tool Use and Agent; benchmark label TOEFL" }, "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240123_introducing_qwen_qwen_toefl" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "release_id": "release_qwen_20240123_introducing_qwen_qwen", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240123_introducing_qwen_qwen", "source_revision_id": "revision_qwen_revision_qwen_20240123_introducing_qwen_qwen", "source_type": "launch_page", "summary": "The official Qwen release reporting names TOEFL for the released Qwen model." }, { "benchmark_id": "benchmark_aitz", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_aitz", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label AITZ" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_aitz" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names AITZ for the released Qwen model." }, { "benchmark_id": "benchmark_alfred", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_alfred", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label ALFRED" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_alfred" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ALFRED for the released Qwen model." }, { "benchmark_id": "benchmark_chartqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_chartqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label ChartQA" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ChartQA for the released Qwen model." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_docvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label DocVQA" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names DocVQA for the released Qwen model." }, { "benchmark_id": "benchmark_egoschema", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_egoschema", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label EgoSchema" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_egoschema" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names EgoSchema for the released Qwen model." }, { "benchmark_id": "benchmark_fncall", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_fncall", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label FnCall" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_fncall" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names FnCall for the released Qwen model." }, { "benchmark_id": "benchmark_gym_cards", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_gym_cards", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label Gym Cards" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_gym_cards" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names Gym Cards for the released Qwen model." }, { "benchmark_id": "benchmark_hallbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_hallbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label HallBench" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_hallbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names HallBench for the released Qwen model." }, { "benchmark_id": "benchmark_infographicvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_infovqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label InfoVQA" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_infovqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names InfoVQA for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MathVision" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MathVista" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVista for the released Qwen model." }, { "benchmark_id": "benchmark_mmbench_1_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMBench 1.1" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmbench_1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBench 1.1 for the released Qwen model." }, { "benchmark_id": "benchmark_mme", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mme", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MME" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mme" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MME for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMMU" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmstar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMStar" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmstar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMStar for the released Qwen model." }, { "benchmark_id": "benchmark_mmt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMT-Bench" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_mmvet", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmvet", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMVet" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mmvet" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMVet for the released Qwen model." }, { "benchmark_id": "benchmark_mtvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mtvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MTVQA" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mtvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MTVQA for the released Qwen model." }, { "benchmark_id": "benchmark_mvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MVBench" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_mvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MVBench for the released Qwen model." }, { "benchmark_id": "benchmark_ocrbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label OCRBench" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_ocrbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCRBench for the released Qwen model." }, { "benchmark_id": "benchmark_perception_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_perceptiontest", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label PerceptionTest" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_perceptiontest" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names PerceptionTest for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label RealWorldQA" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_textvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_textvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label TextVQA" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_textvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names TextVQA for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_with_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-enabled evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label VideoMME with subtitles" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_videomme_with_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME with subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_without_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-free evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label VideoMME without subtitles" }, "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240125_introducing_qwen_vl_qwen_vl_videomme_without_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "release_id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_revision_id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME without subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_alpacaeval_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_alpacaeval_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label AlpacaEval 2.0" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_alpacaeval_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names AlpacaEval 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label BBH" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names BBH for the released Qwen model." }, { "benchmark_id": "benchmark_belebele", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_belebele", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label BELEBELE" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_belebele" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names BELEBELE for the released Qwen model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label C-Eval" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names C-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_cmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_cmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label CMMLU" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_cmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names CMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_coursera", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_coursera", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label Coursera" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_coursera" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Coursera for the released Qwen model." }, { "benchmark_id": "benchmark_flores", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_flores", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label FLORES" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_flores" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names FLORES for the released Qwen model." }, { "benchmark_id": "benchmark_gsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_gsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label GSM" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_gsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_m3exams", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_m3exams", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label M3Exams" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_m3exams" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names M3Exams for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label MGSM" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MGSM for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label MMLU" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_multi", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mmlu_multi", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label MMLU-multi" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mmlu_multi" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-multi for the released Qwen model." }, { "benchmark_id": "benchmark_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label MT-Bench" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_paws_x", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_paws_x", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label PAWS-X" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_paws_x" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names PAWS-X for the released Qwen model." }, { "benchmark_id": "benchmark_quality", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_quality", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label QuALITY" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_quality" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names QuALITY for the released Qwen model." }, { "benchmark_id": "benchmark_sfiction", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_sfiction", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label SFiction" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_sfiction" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names SFiction for the released Qwen model." }, { "benchmark_id": "benchmark_toefl", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_toefl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label TOEFL" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_toefl" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names TOEFL for the released Qwen model." }, { "benchmark_id": "benchmark_xcopa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xcopa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label XCOPA" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xcopa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names XCOPA for the released Qwen model." }, { "benchmark_id": "benchmark_xstorycloze", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xstorycloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label XStoryCloze" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xstorycloze" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names XStoryCloze for the released Qwen model." }, { "benchmark_id": "benchmark_xwinograd", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xwinograd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Models and Chat Models; benchmark label XWinograd" }, "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240204_introducing_qwen1_5_qwen1_5_xwinograd" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "release_id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names XWinograd for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in section Performance; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "source_revision_id": "revision_qwen_revision_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in section Performance; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "source_revision_id": "revision_qwen_revision_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in section Performance; benchmark label MMLU" }, "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "source_revision_id": "revision_qwen_revision_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in section Performance; benchmark label MT-Bench" }, "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "release_id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "source_revision_id": "revision_qwen_revision_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "source_type": "launch_page", "summary": "The official Qwen release reporting names MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_alpacaeval_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_alpacaeval_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Chat Model; benchmark label AlpacaEval 2.0" }, "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_alpacaeval_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_revision_id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names AlpacaEval 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Chat Model; benchmark label BBH" }, "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_revision_id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names BBH for the released Qwen model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Chat Model; benchmark label C-Eval" }, "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_revision_id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names C-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_cmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_cmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Chat Model; benchmark label CMMLU" }, "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_cmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_revision_id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names CMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Chat Model; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_revision_id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Chat Model; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_revision_id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Chat Model; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_revision_id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Chat Model; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_revision_id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Chat Model; benchmark label MMLU" }, "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_revision_id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Chat Model; benchmark label MT-Bench" }, "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "release_id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_revision_id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at evaluation table under heading Performance; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "release_id": "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval_plus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_humaneval_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at evaluation table under heading Performance; benchmark label HumanEval Plus" }, "model_id": "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_humaneval_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "release_id": "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval Plus for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at evaluation table under heading Performance; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "release_id": "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp_plus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_mbpp_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at evaluation table under heading Performance; benchmark label MBPP Plus" }, "model_id": "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5_mbpp_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "release_id": "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "source_revision_id": "revision_qwen_revision_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP Plus for the released Qwen model." }, { "benchmark_id": "benchmark_alpacaeval_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_alpacaeval_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label AlpacaEval 2.0" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_alpacaeval_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names AlpacaEval 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_arc_c", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label ARC-C" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_arc_c" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names ARC-C for the released Qwen model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label BBH" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names BBH for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_hellaswag", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label HellaSwag" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HellaSwag for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label MMLU" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label MT-Bench" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_theoremqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_theoremqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables under heading Performance; benchmark label TheoremQA" }, "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b_theoremqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "release_id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_revision_id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "source_type": "launch_page", "summary": "The official Qwen release reporting names TheoremQA for the released Qwen model." }, { "benchmark_id": "benchmark_arena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428_arena", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_max_text_general", "locator": { "kind": "figure", "value": "Figure or source text at section Evaluation Results; benchmark label Arena" }, "model_id": "model_qwen_model_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-11", "release_id": "release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "source_revision_id": "revision_qwen_revision_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena for the released Qwen model." }, { "benchmark_id": "benchmark_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_max_text_general", "locator": { "kind": "figure", "value": "Figure or source text at section Evaluation Results; benchmark label MT-Bench" }, "model_id": "model_qwen_model_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-11", "release_id": "release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "source_revision_id": "revision_qwen_revision_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "source_type": "launch_page", "summary": "The official Qwen release reporting names MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_alignbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_alignbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label AlignBench" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_alignbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names AlignBench for the released Qwen model." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_arc_c", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label ARC-C" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_arc_c" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names ARC-C for the released Qwen model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label Arena-Hard" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena-Hard for the released Qwen model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label BBH" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names BBH for the released Qwen model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label C-Eval" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names C-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_cmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_cmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label CMMLU" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_cmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names CMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_evalplus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label EvalPlus" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_evalplus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names EvalPlus for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_hellaswag", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label HellaSwag" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names HellaSwag for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label IFEval" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_m_mmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_m_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label M-MMLU" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_m_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names M-MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label MGSM" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names MGSM for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label MMLU" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label MT-Bench" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_multi_exam", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label Multi-Exam" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names Multi-Exam for the released Qwen model." }, { "benchmark_id": "benchmark_multi_mathematics", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_mathematics", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label Multi-Mathematics" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_mathematics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names Multi-Mathematics for the released Qwen model." }, { "benchmark_id": "benchmark_multi_translation", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_translation", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label Multi-Translation" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_translation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names Multi-Translation for the released Qwen model." }, { "benchmark_id": "benchmark_multi_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label Multi-Understanding" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multi_understanding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names Multi-Understanding for the released Qwen model." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label MultiPL-E" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_multipl_e" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiPL-E for the released Qwen model." }, { "benchmark_id": "benchmark_theoremqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_theoremqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label TheoremQA" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_theoremqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names TheoremQA for the released Qwen model." }, { "benchmark_id": "benchmark_truthfulqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_truthfulqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label TruthfulQA" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_truthfulqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names TruthfulQA for the released Qwen model." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_winogrande", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables in sections Base Language Model and Instruction-Tuned Language Model; benchmark label Winogrande" }, "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240606_hello_qwen2_qwen2_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "release_id": "release_qwen_20240606_hello_qwen2_qwen2", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "source_revision_id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "source_type": "launch_page", "summary": "The official Qwen release reporting names Winogrande for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label AIME 2024" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2024 for the released Qwen model." }, { "benchmark_id": "benchmark_amc_2023", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_amc_2023", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label AMC 2023" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_amc_2023" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names AMC 2023 for the released Qwen model." }, { "benchmark_id": "benchmark_cmath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_cmath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label CMATH" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_cmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names CMATH for the released Qwen model." }, { "benchmark_id": "benchmark_cn_middle_school_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_cn_middle_school_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label CN Middle School 2024" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_cn_middle_school_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names CN Middle School 2024 for the released Qwen model." }, { "benchmark_id": "benchmark_collegemath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_collegemath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label CollegeMath" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_collegemath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names CollegeMath for the released Qwen model." }, { "benchmark_id": "benchmark_gaokao", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label GaoKao" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names GaoKao for the released Qwen model." }, { "benchmark_id": "benchmark_gaokao_math_cloze", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao_math_cloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label GaoKao Math Cloze" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao_math_cloze" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names GaoKao Math Cloze for the released Qwen model." }, { "benchmark_id": "benchmark_gaokao_math_qa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao_math_qa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label GaoKao Math QA" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gaokao_math_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names GaoKao Math QA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_stem", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label MMLU-STEM" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_mmlu_stem" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-STEM for the released Qwen model." }, { "benchmark_id": "benchmark_olympiadbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2-Math Base Models and Instruction-Tuned Models; benchmark label OlympiadBench" }, "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240807_introducing_qwen2_math_qwen2_math_olympiadbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "release_id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_revision_id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names OlympiadBench for the released Qwen model." }, { "benchmark_id": "benchmark_aitz", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_aitz", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label AITZ" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_aitz" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names AITZ for the released Qwen model." }, { "benchmark_id": "benchmark_alfred", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_alfred", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label ALFRED" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_alfred" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ALFRED for the released Qwen model." }, { "benchmark_id": "benchmark_chartqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_chartqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label ChartQA" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_chartqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ChartQA for the released Qwen model." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_docvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label DocVQA" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names DocVQA for the released Qwen model." }, { "benchmark_id": "benchmark_egoschema", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_egoschema", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label EgoSchema" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_egoschema" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names EgoSchema for the released Qwen model." }, { "benchmark_id": "benchmark_fncall", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_fncall", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label FnCall" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_fncall" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names FnCall for the released Qwen model." }, { "benchmark_id": "benchmark_gym_cards", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_gym_cards", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label Gym Cards" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_gym_cards" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names Gym Cards for the released Qwen model." }, { "benchmark_id": "benchmark_hallbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_hallbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label HallBench" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_hallbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names HallBench for the released Qwen model." }, { "benchmark_id": "benchmark_infographicvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_infovqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label InfoVQA" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_infovqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names InfoVQA for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MathVision" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MathVista" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVista for the released Qwen model." }, { "benchmark_id": "benchmark_mmbench_1_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMBench 1.1" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmbench_1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBench 1.1 for the released Qwen model." }, { "benchmark_id": "benchmark_mme", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mme", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MME" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mme" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MME for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMMU" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmstar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMStar" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmstar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMStar for the released Qwen model." }, { "benchmark_id": "benchmark_mmt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMT-Bench" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_mmvet", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmvet", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMVet" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mmvet" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMVet for the released Qwen model." }, { "benchmark_id": "benchmark_mtvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mtvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MTVQA" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mtvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MTVQA for the released Qwen model." }, { "benchmark_id": "benchmark_mvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MVBench" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_mvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MVBench for the released Qwen model." }, { "benchmark_id": "benchmark_ocrbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label OCRBench" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_ocrbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCRBench for the released Qwen model." }, { "benchmark_id": "benchmark_perception_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_perceptiontest", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label PerceptionTest" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_perceptiontest" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names PerceptionTest for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label RealWorldQA" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_textvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_textvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label TextVQA" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_textvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names TextVQA for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_with_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-enabled evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label VideoMME with subtitles" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_videomme_with_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME with subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_without_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-free evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label VideoMME without subtitles" }, "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl_videomme_without_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "release_id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_revision_id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME without subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_aider", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label Aider" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_aider" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Aider for the released Qwen model." }, { "benchmark_id": "benchmark_alignbench_1_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_alignbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label AlignBench 1.1" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_alignbench_1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names AlignBench 1.1 for the released Qwen model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label Arena-Hard" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena-Hard for the released Qwen model." }, { "benchmark_id": "benchmark_bigcodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_bigcodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label BigCodeBench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_bigcodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names BigCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_bird_sql", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_bird_sql", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label BIRD-SQL" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_bird_sql" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names BIRD-SQL for the released Qwen model." }, { "benchmark_id": "benchmark_cruxeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_cruxeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label CRUXEval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_cruxeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names CRUXEval for the released Qwen model." }, { "benchmark_id": "benchmark_evalplus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label EvalPlus" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_evalplus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names EvalPlus for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label IFEval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_livebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label LiveBench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_livebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mceval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mceval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label McEval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mceval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names McEval for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label MT-Bench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label MultiPL-E" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_multipl_e" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiPL-E for the released Qwen model." }, { "benchmark_id": "benchmark_spider", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_spider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at performance figures in sections Qwen2.5, Qwen2.5-Coder, and Qwen2.5-Math; benchmark label Spider" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5_spider" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Spider for the released Qwen model." }, { "benchmark_id": "benchmark_aider", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label Aider" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_aider" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names Aider for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label AIME 2024" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2024 for the released Qwen model." }, { "benchmark_id": "benchmark_amc_2023", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_amc_2023", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label AMC 2023" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_amc_2023" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names AMC 2023 for the released Qwen model." }, { "benchmark_id": "benchmark_bigcodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_bigcodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label BigCodeBench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_bigcodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names BigCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_bird_sql", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_bird_sql", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label BIRD-SQL" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_bird_sql" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names BIRD-SQL for the released Qwen model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label C-Eval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names C-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_collegemath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_collegemath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label CollegeMath" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_collegemath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names CollegeMath for the released Qwen model." }, { "benchmark_id": "benchmark_cruxeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_cruxeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label CRUXEval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_cruxeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names CRUXEval for the released Qwen model." }, { "benchmark_id": "benchmark_evalplus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label EvalPlus" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_evalplus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names EvalPlus for the released Qwen model." }, { "benchmark_id": "benchmark_gaokao_2023_english", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gaokao_2023_english", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label GaoKao 2023 English" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gaokao_2023_english" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names GaoKao 2023 English for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label IFEval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mceval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mceval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label McEval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mceval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names McEval for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label MMLU" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label MultiPL-E" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_multipl_e" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiPL-E for the released Qwen model." }, { "benchmark_id": "benchmark_olympiadbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label OlympiadBench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_olympiadbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names OlympiadBench for the released Qwen model." }, { "benchmark_id": "benchmark_spider", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_spider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at performance figures and evaluation table; benchmark label Spider" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder_spider" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names Spider for the released Qwen model." }, { "benchmark_id": "benchmark_alignbench_1_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_alignbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label AlignBench 1.1" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_alignbench_1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names AlignBench 1.1 for the released Qwen model." }, { "benchmark_id": "benchmark_ammlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_ammlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label AMMLU" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_ammlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names AMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_arc_c", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_arc_c", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label ARC-C" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_arc_c" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names ARC-C for the released Qwen model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label Arena-Hard" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena-Hard for the released Qwen model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label BBH" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names BBH for the released Qwen model." }, { "benchmark_id": "benchmark_blend", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_blend", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label BLEnD" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_blend" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names BLEnD for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_hellaswag", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_hellaswag", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label HellaSwag" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_hellaswag" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names HellaSwag for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval_plus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_humaneval_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label HumanEval Plus" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_humaneval_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval Plus for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label IFEval" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_indommlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_indommlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label IndoMMLU" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_indommlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names IndoMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_jmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_jmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label JMMLU" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_jmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names JMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_kmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_kmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label KMMLU" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_kmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names KMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_livebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label LiveBench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_livebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp_plus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mbpp_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MBPP Plus" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mbpp_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP Plus for the released Qwen model." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MGSM" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MGSM for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MMLU" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MMLU Multilingual" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_stem", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MMLU-STEM" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mmlu_stem" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-STEM for the released Qwen model." }, { "benchmark_id": "benchmark_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MT-Bench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_multi_exam", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label Multi-Exam" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names Multi-Exam for the released Qwen model." }, { "benchmark_id": "benchmark_multi_mathematics", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_mathematics", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label Multi-Mathematics" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_mathematics" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names Multi-Mathematics for the released Qwen model." }, { "benchmark_id": "benchmark_multi_translation", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_translation", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label Multi-Translation" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_translation" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names Multi-Translation for the released Qwen model." }, { "benchmark_id": "benchmark_multi_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_understanding", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label Multi-Understanding" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multi_understanding" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names Multi-Understanding for the released Qwen model." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label MultiPL-E" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_multipl_e" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiPL-E for the released Qwen model." }, { "benchmark_id": "benchmark_okapi_mmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_okapi_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label Okapi MMLU" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_okapi_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names Okapi MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_theoremqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_theoremqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label TheoremQA" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_theoremqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names TheoremQA for the released Qwen model." }, { "benchmark_id": "benchmark_truthfulqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_truthfulqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label TruthfulQA" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_truthfulqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names TruthfulQA for the released Qwen model." }, { "benchmark_id": "benchmark_turkishmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_turkishmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label TurkishMMLU" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_turkishmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names TurkishMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_winogrande", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_winogrande", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at evaluation tables throughout the technical release article; benchmark label Winogrande" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm_winogrande" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "source_type": "launch_page", "summary": "The official Qwen release reporting names Winogrande for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label AIME 2024" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2024 for the released Qwen model." }, { "benchmark_id": "benchmark_amc_2023", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_amc_2023", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label AMC 2023" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_amc_2023" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names AMC 2023 for the released Qwen model." }, { "benchmark_id": "benchmark_cmath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_cmath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label CMATH" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_cmath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names CMATH for the released Qwen model." }, { "benchmark_id": "benchmark_cn_middle_school_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_cn_middle_school_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label CN Middle School 2024" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_cn_middle_school_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names CN Middle School 2024 for the released Qwen model." }, { "benchmark_id": "benchmark_collegemath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_collegemath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label CollegeMath" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_collegemath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names CollegeMath for the released Qwen model." }, { "benchmark_id": "benchmark_gaokao", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label GaoKao" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names GaoKao for the released Qwen model." }, { "benchmark_id": "benchmark_gaokao_math_cloze", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao_math_cloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label GaoKao Math Cloze" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao_math_cloze" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names GaoKao Math Cloze for the released Qwen model." }, { "benchmark_id": "benchmark_gaokao_math_qa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao_math_qa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label GaoKao Math QA" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gaokao_math_qa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names GaoKao Math QA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_minerva_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_minerva_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label Minerva Math" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_minerva_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names Minerva Math for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_stem", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_mmlu_stem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label MMLU-STEM" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_mmlu_stem" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-STEM for the released Qwen model." }, { "benchmark_id": "benchmark_olympiadbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen2.5-Math Base Models and Instruction-Tuned Models; benchmark label OlympiadBench" }, "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math_olympiadbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "release_id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "review_status": "verified", "source_id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_revision_id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "source_type": "launch_page", "summary": "The official Qwen release reporting names OlympiadBench for the released Qwen model." }, { "benchmark_id": "benchmark_aider", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label Aider" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_aider" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names Aider for the released Qwen model." }, { "benchmark_id": "benchmark_bigcodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_bigcodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label BigCodeBench" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_bigcodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names BigCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_code_arena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_code_arena", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label Code Arena" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_code_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names Code Arena for the released Qwen model." }, { "benchmark_id": "benchmark_crosscodeeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_crosscodeeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label CrossCodeEval" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_crosscodeeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names CrossCodeEval for the released Qwen model." }, { "benchmark_id": "benchmark_crosscodelongeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_crosscodelongeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label CrossCodeLongEval" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_crosscodelongeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names CrossCodeLongEval for the released Qwen model." }, { "benchmark_id": "benchmark_evalplus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label EvalPlus" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_evalplus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names EvalPlus for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval_infilling", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_humaneval_infilling", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label HumanEval-Infilling" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_humaneval_infilling" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval-Infilling for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mceval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mceval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label McEval" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mceval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names McEval for the released Qwen model." }, { "benchmark_id": "benchmark_mdeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mdeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label MdEval" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_mdeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names MdEval for the released Qwen model." }, { "benchmark_id": "benchmark_repoeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_repoeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label RepoEval" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_repoeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names RepoEval for the released Qwen model." }, { "benchmark_id": "benchmark_safim", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_safim", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at sections Powerful and Practical; benchmark label SAFIM" }, "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family_safim" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "release_id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_revision_id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "source_type": "launch_page", "summary": "The official Qwen release reporting names SAFIM for the released Qwen model." }, { "benchmark_id": "benchmark_alignbench_1_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_alignbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label AlignBench 1.1" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_alignbench_1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names AlignBench 1.1 for the released Qwen model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label Arena-Hard" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena-Hard for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label IFEval" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_livebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label LiveBench" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_livebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveBench for the released Qwen model." }, { "benchmark_id": "benchmark_longbench_chat", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_longbench_chat", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label LongBench-Chat" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_longbench_chat" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names LongBench-Chat for the released Qwen model." }, { "benchmark_id": "benchmark_lv_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_lv_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label LV-Eval" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_lv_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names LV-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_passkey_retrieval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_passkey_retrieval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label Passkey Retrieval" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_passkey_retrieval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names Passkey Retrieval for the released Qwen model." }, { "benchmark_id": "benchmark_ruler", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_ruler", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Passkey Retrieval, More Complex Long Text Tasks, and Short Text Tasks; benchmark label RULER" }, "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo_ruler" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "release_id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_revision_id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "source_type": "launch_page", "summary": "The official Qwen release reporting names RULER for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label AIME 2024" }, "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "source_revision_id": "revision_qwen_revision_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2024 for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "source_revision_id": "revision_qwen_revision_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "source_revision_id": "revision_qwen_revision_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_math_500", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_math_500", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label MATH-500" }, "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview_math_500" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "release_id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "source_revision_id": "revision_qwen_revision_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH-500 for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label MathVision" }, "model_id": "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "release_id": "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "source_revision_id": "revision_qwen_revision_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label MathVista" }, "model_id": "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "release_id": "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "source_revision_id": "revision_qwen_revision_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVista for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label MMMU" }, "model_id": "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "release_id": "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "source_revision_id": "revision_qwen_revision_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_olympiadbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_olympiadbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label OlympiadBench" }, "model_id": "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview_olympiadbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "release_id": "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "source_revision_id": "revision_qwen_revision_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names OlympiadBench for the released Qwen model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label Arena-Hard" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena-Hard for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label IFEval" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_livebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label LiveBench" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_livebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_longbench_chat", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_longbench_chat", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label LongBench-Chat" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_longbench_chat" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names LongBench-Chat for the released Qwen model." }, { "benchmark_id": "benchmark_lv_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_lv_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label LV-Eval" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_lv_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names LV-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label MT-Bench" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label MultiPL-E" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_multipl_e" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiPL-E for the released Qwen model." }, { "benchmark_id": "benchmark_passkey_retrieval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_passkey_retrieval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label Passkey Retrieval" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_passkey_retrieval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names Passkey Retrieval for the released Qwen model." }, { "benchmark_id": "benchmark_ruler", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_ruler", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Long-Context Tasks and Short-Context Tasks; benchmark label RULER" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m_ruler" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "source_type": "launch_page", "summary": "The official Qwen release reporting names RULER for the released Qwen model." }, { "benchmark_id": "benchmark_aitz", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_aitz", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label AITZ" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_aitz" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names AITZ for the released Qwen model." }, { "benchmark_id": "benchmark_android_control", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_android_control", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label Android Control" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_android_control" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names Android Control for the released Qwen model." }, { "benchmark_id": "benchmark_androidworld", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label AndroidWorld" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_androidworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names AndroidWorld for the released Qwen model." }, { "benchmark_id": "benchmark_cc_ocr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label CC-OCR" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_cc_ocr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names CC-OCR for the released Qwen model." }, { "benchmark_id": "benchmark_charadessta", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_charadessta", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label CharadesSTA" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_charadessta" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharadesSTA for the released Qwen model." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_docvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label DocVQA" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names DocVQA for the released Qwen model." }, { "benchmark_id": "benchmark_infographicvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_infovqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label InfoVQA" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_infovqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names InfoVQA for the released Qwen model." }, { "benchmark_id": "benchmark_lvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label LVBench" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_lvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names LVBench for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MathVision" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MathVista" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVista for the released Qwen model." }, { "benchmark_id": "benchmark_megabench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_megabench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MegaBench" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_megabench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MegaBench for the released Qwen model." }, { "benchmark_id": "benchmark_mmbench_1_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmbench_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MMBench 1.1" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmbench_1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBench 1.1 for the released Qwen model." }, { "benchmark_id": "benchmark_mmbench_video", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmbench_video", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MMBench-Video" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmbench_video" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBench-Video for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MMMU" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmstar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MMStar" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_mmstar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMStar for the released Qwen model." }, { "benchmark_id": "benchmark_ocrbenchv2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_ocrbenchv2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label OCRBenchV2" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_ocrbenchv2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCRBenchV2 for the released Qwen model." }, { "benchmark_id": "benchmark_osworld", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_osworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label OSWorld" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_osworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names OSWorld for the released Qwen model." }, { "benchmark_id": "benchmark_screenspot", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_screenspot", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label ScreenSpot" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_screenspot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ScreenSpot for the released Qwen model." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label ScreenSpot Pro" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ScreenSpot Pro for the released Qwen model." }, { "benchmark_id": "benchmark_videomme", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_videomme", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label VideoMME" }, "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_videomme" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "release_id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_revision_id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME for the released Qwen model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label Arena-Hard" }, "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "source_revision_id": "revision_qwen_revision_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena-Hard for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label GPQA-Diamond" }, "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "source_revision_id": "revision_qwen_revision_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA-Diamond for the released Qwen model." }, { "benchmark_id": "benchmark_livebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label LiveBench" }, "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_livebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "source_revision_id": "revision_qwen_revision_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "source_revision_id": "revision_qwen_revision_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at section Performance; benchmark label MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "release_id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "source_revision_id": "revision_qwen_revision_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label AIME 2024" }, "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "source_revision_id": "revision_qwen_revision_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2024 for the released Qwen model." }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_bfcl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label BFCL" }, "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_bfcl" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "source_revision_id": "revision_qwen_revision_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names BFCL for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label IFEval" }, "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "source_revision_id": "revision_qwen_revision_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_livebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label LiveBench" }, "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_livebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "source_revision_id": "revision_qwen_revision_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "release_id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "source_revision_id": "revision_qwen_revision_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_docvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_docvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label DocVQA" }, "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_docvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_revision_id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names DocVQA for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label GPQA-Diamond" }, "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_revision_id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA-Diamond for the released Qwen model." }, { "benchmark_id": "benchmark_humaneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_humaneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label HumanEval" }, "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_humaneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_revision_id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HumanEval for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_revision_id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MathVista" }, "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_revision_id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVista for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_revision_id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mm_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mm_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MM-MT-Bench" }, "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mm_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_revision_id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MM-MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_revision_id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MMMU" }, "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_revision_id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "release_id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_revision_id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_common_voice", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_common_voice", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure and text under heading Performance; benchmark label Common Voice" }, "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_common_voice" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names Common Voice for the released Qwen model." }, { "benchmark_id": "benchmark_covost2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_covost2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure and text under heading Performance; benchmark label CoVoST2" }, "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_covost2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names CoVoST2 for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure and text under heading Performance; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_mmau", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmau", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure and text under heading Performance; benchmark label MMAU" }, "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmau" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMAU for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure and text under heading Performance; benchmark label MMLU" }, "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure and text under heading Performance; benchmark label MMMU" }, "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmstar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure and text under heading Performance; benchmark label MMStar" }, "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mmstar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMStar for the released Qwen model." }, { "benchmark_id": "benchmark_mvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure and text under heading Performance; benchmark label MVBench" }, "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_mvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MVBench for the released Qwen model." }, { "benchmark_id": "benchmark_omnibench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_omnibench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure and text under heading Performance; benchmark label OmniBench" }, "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_omnibench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names OmniBench for the released Qwen model." }, { "benchmark_id": "benchmark_seed_tts_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_seed_tts_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure and text under heading Performance; benchmark label Seed-TTS-Eval" }, "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni_seed_tts_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "release_id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names Seed-TTS-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "locator": { "kind": "figure", "value": "Figure or source text at section Introduction; benchmark label MathVision" }, "model_id": "model_qwen_model_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-27", "release_id": "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_aider", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label Aider" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aider" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names Aider for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2024", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aime_2024", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label AIME 2024" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aime_2024" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2024 for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label AIME 2025" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_arena_hard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_arena_hard", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label Arena-Hard" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_arena_hard" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena-Hard for the released Qwen model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label BBH" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names BBH for the released Qwen model." }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_bfcl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label BFCL" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_bfcl" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names BFCL for the released Qwen model." }, { "benchmark_id": "benchmark_codeforces", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_codeforces", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label CodeForces" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_codeforces" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names CodeForces for the released Qwen model." }, { "benchmark_id": "benchmark_crux_o", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_crux_o", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label CRUX-O" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_crux_o" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names CRUX-O for the released Qwen model." }, { "benchmark_id": "benchmark_evalplus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label EvalPlus" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_evalplus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names EvalPlus for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label INCLUDE" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names INCLUDE for the released Qwen model." }, { "benchmark_id": "benchmark_livebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label LiveBench" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_livebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mbpp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mbpp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label MBPP" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mbpp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names MBPP for the released Qwen model." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label MGSM" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names MGSM for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label MMLU" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label MMMLU" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_multiif", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_multiif", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label MultiIF" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_multiif" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiIF for the released Qwen model." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label MultiPL-E" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_multipl_e" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiPL-E for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Introduction and Pre-training; benchmark label SuperGPQA" }, "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250428_qwen3_think_deeper_act_faster_qwen3_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "release_id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_revision_id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at section Scaling Long-Horizon RL; benchmark label SWE-bench Verified" }, "model_id": "model_qwen_model_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "release_id": "release_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "source_revision_id": "revision_qwen_revision_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Verified for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label AIME 2025" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_arena_hard_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_arena_hard_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label Arena-Hard v2" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_arena_hard_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena-Hard v2 for the released Qwen model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label BBH" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names BBH for the released Qwen model." }, { "benchmark_id": "benchmark_crux_o", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_crux_o", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label CRUX-O" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_crux_o" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names CRUX-O for the released Qwen model." }, { "benchmark_id": "benchmark_evalplus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label EvalPlus" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_evalplus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names EvalPlus for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label GSM8K" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label INCLUDE" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names INCLUDE for the released Qwen model." }, { "benchmark_id": "benchmark_livebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label LiveBench" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_livebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label MATH" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label MGSM" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MGSM for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label MMLU" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label MMMLU" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_ruler", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_ruler", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label RULER" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_ruler" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names RULER for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under headings Pre-training and Post-training; benchmark label SuperGPQA" }, "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "release_id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_revision_id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label AIME 2025" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_fleurs", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_fleurs", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label FLEURS" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_fleurs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names FLEURS for the released Qwen model." }, { "benchmark_id": "benchmark_gtzan", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_gtzan", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label GTZAN" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_gtzan" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names GTZAN for the released Qwen model." }, { "benchmark_id": "benchmark_hallusionbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label HallusionBench" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_hallusionbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names HallusionBench for the released Qwen model." }, { "benchmark_id": "benchmark_librispeech", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_librispeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label LibriSpeech" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_librispeech" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names LibriSpeech for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MathVision" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_minimax_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_minimax_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MiniMax Multilingual" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_minimax_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MiniMax Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_mir_1k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mir_1k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MIR-1K" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mir_1k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MIR-1K for the released Qwen model." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MLVU" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLVU for the released Qwen model." }, { "benchmark_id": "benchmark_mmau", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mmau", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MMAU" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mmau" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMAU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_rul_muchomusic", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_rul_muchomusic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label RUL-MuchoMusic" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_rul_muchomusic" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names RUL-MuchoMusic for the released Qwen model." }, { "benchmark_id": "benchmark_seed_tts_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_seed_tts_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label Seed-TTS-Eval" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_seed_tts_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names Seed-TTS-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_voicebench_commoneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_voicebench_commoneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label VoiceBench CommonEval" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_voicebench_commoneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names VoiceBench CommonEval for the released Qwen model." }, { "benchmark_id": "benchmark_wenetspeech", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_wenetspeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label WenetSpeech" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_wenetspeech" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names WenetSpeech for the released Qwen model." }, { "benchmark_id": "benchmark_worldsense", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_worldsense", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label WorldSense" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_worldsense" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names WorldSense for the released Qwen model." }, { "benchmark_id": "benchmark_writingbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_writingbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label WritingBench" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_writingbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names WritingBench for the released Qwen model." }, { "benchmark_id": "benchmark_zebralogic", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_zebralogic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Performance; benchmark label ZebraLogic" }, "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni_zebralogic" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "release_id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_revision_id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZebraLogic for the released Qwen model." }, { "benchmark_id": "benchmark_ai2d_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label AI2D Test" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ai2d_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names AI2D Test for the released Qwen model." }, { "benchmark_id": "benchmark_androidworld", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label AndroidWorld" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_androidworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names AndroidWorld for the released Qwen model." }, { "benchmark_id": "benchmark_arena_hard_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_arena_hard_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label Arena-Hard v2" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_arena_hard_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena-Hard v2 for the released Qwen model." }, { "benchmark_id": "benchmark_arkitscenes", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_arkitscenes", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label ARKitScenes" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_arkitscenes" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ARKitScenes for the released Qwen model." }, { "benchmark_id": "benchmark_berkeley_function_calling_leaderboard", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_bfcl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label BFCL" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_bfcl" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names BFCL for the released Qwen model." }, { "benchmark_id": "benchmark_blink", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_blink", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label BLINK" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_blink" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names BLINK for the released Qwen model." }, { "benchmark_id": "benchmark_cc_ocr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label CC-OCR" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_cc_ocr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names CC-OCR for the released Qwen model." }, { "benchmark_id": "benchmark_charadessta", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_charadessta", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label CharadesSTA" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_charadessta" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharadesSTA for the released Qwen model." }, { "benchmark_id": "benchmark_chartmimic", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_chartmimic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label ChartMimic" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_chartmimic" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ChartMimic for the released Qwen model." }, { "benchmark_id": "benchmark_charxiv_rq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label CharXiv RQ" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_charxiv_rq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharXiv RQ for the released Qwen model." }, { "benchmark_id": "benchmark_countbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label CountBench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_countbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names CountBench for the released Qwen model." }, { "benchmark_id": "benchmark_creative_writing_v3", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_creative_writing_v3", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label Creative Writing v3" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_creative_writing_v3" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names Creative Writing v3 for the released Qwen model." }, { "benchmark_id": "benchmark_csimpleqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_csimpleqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label CSimpleQA" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_csimpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names CSimpleQA for the released Qwen model." }, { "benchmark_id": "benchmark_design2code", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_design2code", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label Design2Code" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_design2code" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names Design2Code for the released Qwen model." }, { "benchmark_id": "benchmark_docvqa_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_docvqa_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label DocVQA Test" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_docvqa_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names DocVQA Test for the released Qwen model." }, { "benchmark_id": "benchmark_embspatialbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label EmbSpatialBench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_embspatialbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names EmbSpatialBench for the released Qwen model." }, { "benchmark_id": "benchmark_erqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label ERQA" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_erqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ERQA for the released Qwen model." }, { "benchmark_id": "benchmark_hallusionbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label HallusionBench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hallusionbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names HallusionBench for the released Qwen model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label HLE" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hmmt_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label HMMT 2025" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hmmt_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_hypersim", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hypersim", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label HyperSim" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_hypersim" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names HyperSim for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label IFEval" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label INCLUDE" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names INCLUDE for the released Qwen model." }, { "benchmark_id": "benchmark_infovqa_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_infovqa_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label InfoVQA Test" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_infovqa_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names InfoVQA Test for the released Qwen model." }, { "benchmark_id": "benchmark_livebench_july_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livebench_july_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label LiveBench July 2025" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livebench_july_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveBench July 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v5", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livecodebench_v5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label LiveCodeBench v5" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livecodebench_v5" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v5 for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_lvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label LVBench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_lvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names LVBench for the released Qwen model." }, { "benchmark_id": "benchmark_mathverse", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathverse", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MathVerse" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathverse" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVerse for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MathVision" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_mathvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MathVista" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mathvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVista for the released Qwen model." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MLVU" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLVU for the released Qwen model." }, { "benchmark_id": "benchmark_mm_mt_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mm_mt_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MM-MT-Bench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mm_mt_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MM-MT-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_mmbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MMBench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBench for the released Qwen model." }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MMBench EN 1.1 dev" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmbench_en_1_1_dev" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBench EN 1.1 dev for the released Qwen model." }, { "benchmark_id": "benchmark_mmlongbench_doc", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlongbench_doc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MMLongBench-Doc" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlongbench_doc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLongBench-Doc for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_prox", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MMLU-ProX" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlu_prox" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-ProX for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MMMU" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmstar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MMStar" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_mmstar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMStar for the released Qwen model." }, { "benchmark_id": "benchmark_muirbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_muirbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MUIRBENCH" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_muirbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MUIRBENCH for the released Qwen model." }, { "benchmark_id": "benchmark_multiif", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_multiif", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MultiIF" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_multiif" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiIF for the released Qwen model." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label MultiPL-E" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_multipl_e" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiPL-E for the released Qwen model." }, { "benchmark_id": "benchmark_objectron", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_objectron", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label Objectron" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_objectron" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names Objectron for the released Qwen model." }, { "benchmark_id": "benchmark_ocrbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label OCRBench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ocrbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCRBench for the released Qwen model." }, { "benchmark_id": "benchmark_ocrbenchv2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ocrbenchv2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label OCRBenchV2" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_ocrbenchv2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCRBenchV2 for the released Qwen model." }, { "benchmark_id": "benchmark_odinw13", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label ODinW13" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_odinw13" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ODinW13 for the released Qwen model." }, { "benchmark_id": "benchmark_osworld", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_osworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label OSWorld" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_osworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names OSWorld for the released Qwen model." }, { "benchmark_id": "benchmark_osworld_g", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_osworld_g", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label OSWorld-G" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_osworld_g" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names OSWorld-G for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label RealWorldQA" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_refcoco", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label RefCOCO" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_refcoco" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefCOCO for the released Qwen model." }, { "benchmark_id": "benchmark_refspatialbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_refspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label RefSpatialBench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_refspatialbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefSpatialBench for the released Qwen model." }, { "benchmark_id": "benchmark_robospatialhome", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_robospatialhome", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label RoboSpatialHome" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_robospatialhome" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names RoboSpatialHome for the released Qwen model." }, { "benchmark_id": "benchmark_screenspot", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_screenspot", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label ScreenSpot" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_screenspot" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ScreenSpot for the released Qwen model." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label ScreenSpot Pro" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ScreenSpot Pro for the released Qwen model." }, { "benchmark_id": "benchmark_sifo", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sifo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label SIFO" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sifo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names SIFO for the released Qwen model." }, { "benchmark_id": "benchmark_sifo_multiturn", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sifo_multiturn", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label SIFO-multiturn" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sifo_multiturn" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names SIFO-multiturn for the released Qwen model." }, { "benchmark_id": "benchmark_simpleqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_simpleqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label SimpleQA" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_simpleqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names SimpleQA for the released Qwen model." }, { "benchmark_id": "benchmark_simplevqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label SimpleVQA" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_simplevqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names SimpleVQA for the released Qwen model." }, { "benchmark_id": "benchmark_sunrgbd", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sunrgbd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label SUNRGBD" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_sunrgbd" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names SUNRGBD for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label SuperGPQA" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_unisvg", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_unisvg", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label UniSVG" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_unisvg" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names UniSVG for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_without_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-free evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label VideoMME without subtitles" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_videomme_without_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME without subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videommmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label VideoMMMU" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_videommmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMMMU for the released Qwen model." }, { "benchmark_id": "benchmark_visulogic", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_visulogic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label VisuLogic" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_visulogic" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names VisuLogic for the released Qwen model." }, { "benchmark_id": "benchmark_vsi_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_vsi_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label VSI-Bench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_vsi_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names VSI-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_writingbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_writingbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label WritingBench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_writingbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names WritingBench for the released Qwen model." }, { "benchmark_id": "benchmark_zerobench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_zerobench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label ZEROBench" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_zerobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZEROBench for the released Qwen model." }, { "benchmark_id": "benchmark_zerobench_sub", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figures under heading Model Performance; benchmark label ZEROBench-Sub" }, "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl_zerobench_sub" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "release_id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_revision_id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZEROBench-Sub for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen3-Max-Instruct and Qwen3-Max-Thinking; benchmark label AIME 2025" }, "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_revision_id": "revision_qwen_revision_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen3-Max-Instruct and Qwen3-Max-Thinking; benchmark label GPQA" }, "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_revision_id": "revision_qwen_revision_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_hmmt_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen3-Max-Instruct and Qwen3-Max-Thinking; benchmark label HMMT 2025" }, "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_hmmt_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_revision_id": "revision_qwen_revision_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen3-Max-Instruct and Qwen3-Max-Thinking; benchmark label LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_revision_id": "revision_qwen_revision_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen3-Max-Instruct and Qwen3-Max-Thinking; benchmark label SuperGPQA" }, "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_revision_id": "revision_qwen_revision_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen3-Max-Instruct and Qwen3-Max-Thinking; benchmark label SWE-bench Verified" }, "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_revision_id": "revision_qwen_revision_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Verified for the released Qwen model." }, { "benchmark_id": "benchmark_tau2_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_tau2_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen3-Max-Instruct and Qwen3-Max-Thinking; benchmark label Tau2-Bench" }, "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_tau2_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_revision_id": "revision_qwen_revision_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names Tau2-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_text_arena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_text_arena", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Qwen3-Max-Instruct and Qwen3-Max-Thinking; benchmark label Text Arena" }, "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20250924_qwen3_max_just_scale_it_qwen3_max_text_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "release_id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "review_status": "verified", "source_id": "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_revision_id": "revision_qwen_revision_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "source_type": "launch_page", "summary": "The official Qwen release reporting names Text Arena for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_aime_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label AIME 2025" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_aime_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_fleurs", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_fleurs", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label FLEURS" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_fleurs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names FLEURS for the released Qwen model." }, { "benchmark_id": "benchmark_gtzan", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_gtzan", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label GTZAN" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_gtzan" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names GTZAN for the released Qwen model." }, { "benchmark_id": "benchmark_livebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_livebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label LiveBench" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_livebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MathVision" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_minimax_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_minimax_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MiniMax Multilingual" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_minimax_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names MiniMax Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_mir_1k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mir_1k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MIR-1K" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mir_1k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names MIR-1K for the released Qwen model." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MLVU" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLVU for the released Qwen model." }, { "benchmark_id": "benchmark_mmau", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmau", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMAU" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmau" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMAU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMMU" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label MultiPL-E" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_multipl_e" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiPL-E for the released Qwen model." }, { "benchmark_id": "benchmark_rul_muchomusic", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_rul_muchomusic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label RUL-MuchoMusic" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_rul_muchomusic" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names RUL-MuchoMusic for the released Qwen model." }, { "benchmark_id": "benchmark_seed_tts_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_seed_tts_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label Seed-TTS-Eval" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_seed_tts_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names Seed-TTS-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_voicebench_commoneval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_voicebench_commoneval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label VoiceBench CommonEval" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_voicebench_commoneval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names VoiceBench CommonEval for the released Qwen model." }, { "benchmark_id": "benchmark_writingbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_writingbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label WritingBench" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_writingbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names WritingBench for the released Qwen model." }, { "benchmark_id": "benchmark_zebralogic", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_zebralogic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "locator": { "kind": "figure", "value": "Figure or source text at figure under heading Performance; benchmark label ZebraLogic" }, "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201_zebralogic" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "release_id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "review_status": "verified", "source_id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_revision_id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZebraLogic for the released Qwen model." }, { "benchmark_id": "benchmark_aa_lcr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_aa_lcr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row AA-LCR" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_aa_lcr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names AA-LCR for the released Qwen model." }, { "benchmark_id": "benchmark_arena_hard_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_arena_hard_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row Arena-Hard v2" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_arena_hard_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena-Hard v2 for the released Qwen model." }, { "benchmark_id": "benchmark_bfcl_v4", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_bfcl_v4", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row BFCL v4" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_bfcl_v4" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names BFCL v4 for the released Qwen model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row C-Eval" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names C-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_deep_planning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row Deep-Planning" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_deep_planning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names Deep-Planning for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row GPQA" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row HLE" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE for the released Qwen model." }, { "benchmark_id": "benchmark_hle_with_tools", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row HLE with tools" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hle_with_tools" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE with tools for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_february_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row HMMT February 2025" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hmmt_february_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT February 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_november_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row HMMT November 2025" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_hmmt_november_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT November 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_ifbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row IFBench" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_ifbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFBench for the released Qwen model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row IMOAnswerBench" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names IMOAnswerBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_multichallenge", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_multichallenge", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row MultiChallenge" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_multichallenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiChallenge for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row SWE-bench Verified" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Verified for the released Qwen model." }, { "benchmark_id": "benchmark_tau2_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_tau2_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row TAU2-Bench" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_tau2_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names TAU2-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_vita_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "locator": { "kind": "table", "value": "Table under heading Introduction; benchmark row VITA-Bench" }, "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260126_qwen3_max_thinking_vita_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "release_id": "release_qwen_20260126_qwen3_max_thinking", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "source_revision_id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "source_type": "launch_page", "summary": "The official Qwen release reporting names VITA-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_aider", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_aider", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at section Agent-Centric Benchmark Results; benchmark label Aider" }, "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_aider" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "release_id": "release_qwen_20260203_qwen3_coder_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260203_qwen3_coder_next", "source_revision_id": "revision_qwen_revision_qwen_20260203_qwen3_coder_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names Aider for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at section Agent-Centric Benchmark Results; benchmark label SWE-bench Multilingual" }, "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "release_id": "release_qwen_20260203_qwen3_coder_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260203_qwen3_coder_next", "source_revision_id": "revision_qwen_revision_qwen_20260203_qwen3_coder_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at section Agent-Centric Benchmark Results; benchmark label SWE-bench Pro" }, "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "release_id": "release_qwen_20260203_qwen3_coder_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260203_qwen3_coder_next", "source_revision_id": "revision_qwen_revision_qwen_20260203_qwen3_coder_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Pro for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at section Agent-Centric Benchmark Results; benchmark label SWE-bench Verified" }, "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "release_id": "release_qwen_20260203_qwen3_coder_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260203_qwen3_coder_next", "source_revision_id": "revision_qwen_revision_qwen_20260203_qwen3_coder_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Verified for the released Qwen model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "locator": { "kind": "figure", "value": "Figure or source text at section Agent-Centric Benchmark Results; benchmark label Terminal-Bench 2.0" }, "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260203_qwen3_coder_next_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "release_id": "release_qwen_20260203_qwen3_coder_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260203_qwen3_coder_next", "source_revision_id": "revision_qwen_revision_qwen_20260203_qwen3_coder_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names Terminal-Bench 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_aa_lcr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_aa_lcr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row AA-LCR" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_aa_lcr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names AA-LCR for the released Qwen model." }, { "benchmark_id": "benchmark_ai2d_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row AI2D Test" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ai2d_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names AI2D Test for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2026", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_aime_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row AIME 2026" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_aime_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2026 for the released Qwen model." }, { "benchmark_id": "benchmark_androidworld", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row AndroidWorld" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_androidworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names AndroidWorld for the released Qwen model." }, { "benchmark_id": "benchmark_babyvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "code interpreter enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_babyvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row BabyVision" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_babyvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names BabyVision for the released Qwen model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Pretraining; benchmark row BBH" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names BBH for the released Qwen model." }, { "benchmark_id": "benchmark_bfcl_v4", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_bfcl_v4", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row BFCL v4" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_bfcl_v4" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names BFCL v4 for the released Qwen model." }, { "benchmark_id": "benchmark_browsecomp", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_browsecomp", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row BrowseComp" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_browsecomp" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names BrowseComp for the released Qwen model." }, { "benchmark_id": "benchmark_browsecomp_zh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_browsecomp_zh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row BrowseComp-zh" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_browsecomp_zh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names BrowseComp-zh for the released Qwen model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row C-Eval" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names C-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_cc_ocr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CC-OCR" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_cc_ocr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names CC-OCR for the released Qwen model." }, { "benchmark_id": "benchmark_charxiv_rq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CharXiv RQ" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_charxiv_rq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharXiv RQ for the released Qwen model." }, { "benchmark_id": "benchmark_countbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CountBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_countbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names CountBench for the released Qwen model." }, { "benchmark_id": "benchmark_crux_i", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_crux_i", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Pretraining; benchmark row CRUX-I" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_crux_i" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names CRUX-I for the released Qwen model." }, { "benchmark_id": "benchmark_crux_o", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_crux_o", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Pretraining; benchmark row CRUX-O" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_crux_o" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names CRUX-O for the released Qwen model." }, { "benchmark_id": "benchmark_deep_planning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Deep-Planning" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_deep_planning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Deep-Planning for the released Qwen model." }, { "benchmark_id": "benchmark_dynamath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_dynamath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row DynaMath" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_dynamath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names DynaMath for the released Qwen model." }, { "benchmark_id": "benchmark_embspatialbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row EmbSpatialBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_embspatialbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names EmbSpatialBench for the released Qwen model." }, { "benchmark_id": "benchmark_erqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ERQA" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_erqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names ERQA for the released Qwen model." }, { "benchmark_id": "benchmark_evalplus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Pretraining; benchmark row EvalPlus" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_evalplus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names EvalPlus for the released Qwen model." }, { "benchmark_id": "benchmark_global_piqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_global_piqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Global PIQA" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_global_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Global PIQA for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row GPQA" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Pretraining; benchmark row GSM8K" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_hallusionbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row HallusionBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hallusionbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names HallusionBench for the released Qwen model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HLE" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE for the released Qwen model." }, { "benchmark_id": "benchmark_hle_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HLE-Verified" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE-Verified for the released Qwen model." }, { "benchmark_id": "benchmark_hle_with_tools", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HLE with tools" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hle_with_tools" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE with tools for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_february_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT February 2025" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hmmt_february_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT February 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_november_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT November 2025" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hmmt_november_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT November 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_hypersim", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hypersim", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row Hypersim" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_hypersim" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Hypersim for the released Qwen model." }, { "benchmark_id": "benchmark_ifbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row IFBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ifbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFBench for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row IFEval" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row IMOAnswerBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names IMOAnswerBench for the released Qwen model." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row INCLUDE" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names INCLUDE for the released Qwen model." }, { "benchmark_id": "benchmark_korbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_korbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Pretraining; benchmark row KoRBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_korbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names KoRBench for the released Qwen model." }, { "benchmark_id": "benchmark_lingoqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_lingoqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row LingoQA" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_lingoqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names LingoQA for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_longbench_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_longbench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row LongBench v2" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_longbench_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names LongBench v2 for the released Qwen model." }, { "benchmark_id": "benchmark_lvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row LVBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_lvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names LVBench for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Pretraining; benchmark row MATH" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "disclosed", "value": "fixed step-by-step prompt" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MathVision" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_mathvista_mini", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mathvista_mini", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MathVista Mini" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mathvista_mini" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVista Mini for the released Qwen model." }, { "benchmark_id": "benchmark_maxife", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_maxife", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MAXIFE" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_maxife" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MAXIFE for the released Qwen model." }, { "benchmark_id": "benchmark_mcp_mark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MCP-Mark" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mcp_mark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MCP-Mark for the released Qwen model." }, { "benchmark_id": "benchmark_medxpertqa_mm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_medxpertqa_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MedXpertQA-MM" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_medxpertqa_mm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MedXpertQA-MM for the released Qwen model." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MLVU" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLVU for the released Qwen model." }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMBench EN 1.1 dev" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmbench_en_1_1_dev" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBench EN 1.1 dev for the released Qwen model." }, { "benchmark_id": "benchmark_mmlongbench_doc", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlongbench_doc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMLongBench-Doc" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlongbench_doc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLongBench-Doc for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Pretraining; benchmark row MMLU" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_prox", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMLU-ProX" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_prox" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-ProX for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMMLU" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMMU" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmstar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMStar" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmstar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMStar for the released Qwen model." }, { "benchmark_id": "benchmark_mmvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMVU" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mmvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMVU for the released Qwen model." }, { "benchmark_id": "benchmark_multichallenge", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_multichallenge", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MultiChallenge" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_multichallenge" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiChallenge for the released Qwen model." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Pretraining; benchmark row MultiPL-E" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_multipl_e" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiPL-E for the released Qwen model." }, { "benchmark_id": "benchmark_mvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MVBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_mvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names MVBench for the released Qwen model." }, { "benchmark_id": "benchmark_nova", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_nova", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row NOVA" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_nova" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names NOVA for the released Qwen model." }, { "benchmark_id": "benchmark_nuscene", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_nuscene", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row Nuscene" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_nuscene" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Nuscene for the released Qwen model." }, { "benchmark_id": "benchmark_ocrbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row OCRBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_ocrbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCRBench for the released Qwen model." }, { "benchmark_id": "benchmark_odinw13", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ODinW13" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_odinw13" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names ODinW13 for the released Qwen model." }, { "benchmark_id": "benchmark_omnidocbench_1_5", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row OmniDocBench 1.5" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_omnidocbench_1_5" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names OmniDocBench 1.5 for the released Qwen model." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_osworld_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row OSWorld Verified" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names OSWorld Verified for the released Qwen model." }, { "benchmark_id": "benchmark_pmc_vqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_pmc_vqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row PMC-VQA" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_pmc_vqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names PMC-VQA for the released Qwen model." }, { "benchmark_id": "benchmark_polymath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_polymath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row PolyMATH" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_polymath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names PolyMATH for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RealWorldQA" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_refcoco", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RefCOCO" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_refcoco" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefCOCO for the released Qwen model." }, { "benchmark_id": "benchmark_refspatialbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_refspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RefSpatialBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_refspatialbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefSpatialBench for the released Qwen model." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ScreenSpot Pro" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names ScreenSpot Pro for the released Qwen model." }, { "benchmark_id": "benchmark_seal_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_seal_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Seal-0" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_seal_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Seal-0 for the released Qwen model." }, { "benchmark_id": "benchmark_seccodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_seccodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SecCodeBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_seccodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names SecCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_simplevqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row SimpleVQA" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_simplevqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names SimpleVQA for the released Qwen model." }, { "benchmark_id": "benchmark_slake", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_slake", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row SLAKE" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_slake" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names SLAKE for the released Qwen model." }, { "benchmark_id": "benchmark_sunrgbd", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_sunrgbd", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row SUNRGBD" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_sunrgbd" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names SUNRGBD for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SuperGPQA" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_swe_agentless", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_agentless", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Pretraining; benchmark row SWE-agentless" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_agentless" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-agentless for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Multilingual" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Verified" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Verified for the released Qwen model." }, { "benchmark_id": "benchmark_tau2_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_tau2_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row TAU2-Bench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_tau2_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names TAU2-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Terminal-Bench 2.0" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Terminal-Bench 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_tool_decathlon", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_tool_decathlon", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Tool-Decathlon" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_tool_decathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names Tool-Decathlon for the released Qwen model." }, { "benchmark_id": "benchmark_v", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "code interpreter enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_v_star", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row V*" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_v_star" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names V* for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_with_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-enabled evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMME with subtitles" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videomme_with_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME with subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_without_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-free evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMME without subtitles" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videomme_without_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME without subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videommmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMMMU" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_videommmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMMMU for the released Qwen model." }, { "benchmark_id": "benchmark_vita_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row VITA-Bench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_vita_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names VITA-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_we_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_we_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row We-Math" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_we_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names We-Math for the released Qwen model." }, { "benchmark_id": "benchmark_widesearch", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row WideSearch" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_widesearch" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names WideSearch for the released Qwen model." }, { "benchmark_id": "benchmark_wmt24", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_wmt24_plus_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row WMT24++" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_wmt24_plus_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names WMT24++ for the released Qwen model." }, { "benchmark_id": "benchmark_zerobench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_zerobench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ZEROBench" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_zerobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZEROBench for the released Qwen model." }, { "benchmark_id": "benchmark_zerobench_sub", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ZEROBench-Sub" }, "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260216_qwen3_5_zerobench_sub" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "release_id": "release_qwen_20260216_qwen3_5", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260216_qwen3_5", "source_revision_id": "revision_qwen_revision_qwen_20260216_qwen3_5", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZEROBench-Sub for the released Qwen model." }, { "benchmark_id": "benchmark_arena", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260319_qwen3_5_max_preview_arena", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_text_general", "locator": { "kind": "figure", "value": "Figure or source text at release introduction; benchmark label Arena" }, "model_id": "model_qwen_model_qwen_20260319_qwen3_5_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260319_qwen3_5_max_preview_arena" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-19", "release_id": "release_qwen_20260319_qwen3_5_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260319_qwen3_5_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260319_qwen3_5_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names Arena for the released Qwen model." }, { "benchmark_id": "benchmark_aa_lcr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_aa_lcr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row AA-LCR" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_aa_lcr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names AA-LCR for the released Qwen model." }, { "benchmark_id": "benchmark_ai2d_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row AI2D Test" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ai2d_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names AI2D Test for the released Qwen model." }, { "benchmark_id": "benchmark_av_speakerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_av_speakerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio-Visual; benchmark row AV-SpeakerBench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_av_speakerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names AV-SpeakerBench for the released Qwen model." }, { "benchmark_id": "benchmark_avut", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_avut", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio-Visual; benchmark row AVUT" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_avut" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names AVUT for the released Qwen model." }, { "benchmark_id": "benchmark_bfcl_v4", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_bfcl_v4", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row BFCL v4" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_bfcl_v4" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names BFCL v4 for the released Qwen model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row C-Eval" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names C-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_cc_ocr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row CC-OCR" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_cc_ocr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names CC-OCR for the released Qwen model." }, { "benchmark_id": "benchmark_charxiv_rq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row CharXiv RQ" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_charxiv_rq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharXiv RQ for the released Qwen model." }, { "benchmark_id": "benchmark_common_voice_15", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_common_voice_15", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row Common Voice 15" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_common_voice_15" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names Common Voice 15 for the released Qwen model." }, { "benchmark_id": "benchmark_countbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row CountBench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_countbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names CountBench for the released Qwen model." }, { "benchmark_id": "benchmark_dailyomni", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_dailyomni", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio-Visual; benchmark row DailyOmni" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_dailyomni" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names DailyOmni for the released Qwen model." }, { "benchmark_id": "benchmark_dynamath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_dynamath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row DynaMath" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_dynamath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names DynaMath for the released Qwen model." }, { "benchmark_id": "benchmark_embspatialbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row EmbSpatialBench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_embspatialbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names EmbSpatialBench for the released Qwen model." }, { "benchmark_id": "benchmark_erqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row ERQA" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_erqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names ERQA for the released Qwen model." }, { "benchmark_id": "benchmark_fleurs", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_fleurs", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row FLEURS" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_fleurs" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names FLEURS for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row GPQA" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_november_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row HMMT November 2025" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_hmmt_november_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT November 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_ifbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row IFBench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ifbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFBench for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row IFEval" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row IMOAnswerBench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names IMOAnswerBench for the released Qwen model." }, { "benchmark_id": "benchmark_kespeech", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_kespeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row Kespeech" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_kespeech" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names Kespeech for the released Qwen model." }, { "benchmark_id": "benchmark_librispeech", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_librispeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row LibriSpeech" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_librispeech" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names LibriSpeech for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_longbench_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_longbench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row LongBench v2" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_longbench_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names LongBench v2 for the released Qwen model." }, { "benchmark_id": "benchmark_lvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row LVBench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_lvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names LVBench for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MathVision" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_mathvista_mini", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mathvista_mini", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MathVista Mini" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mathvista_mini" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVista Mini for the released Qwen model." }, { "benchmark_id": "benchmark_medxpertqa_mm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_medxpertqa_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MedXpertQA-MM" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_medxpertqa_mm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MedXpertQA-MM for the released Qwen model." }, { "benchmark_id": "benchmark_mir_1k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mir_1k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row MIR-1K" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mir_1k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MIR-1K for the released Qwen model." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MLVU" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLVU for the released Qwen model." }, { "benchmark_id": "benchmark_mmar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row MMAR" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMAR for the released Qwen model." }, { "benchmark_id": "benchmark_mmau", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmau", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row MMAU" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmau" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMAU for the released Qwen model." }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MMBench EN 1.1 dev" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmbench_en_1_1_dev" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBench EN 1.1 dev for the released Qwen model." }, { "benchmark_id": "benchmark_mme_videoocr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mme_videoocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MME-VideoOCR" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mme_videoocr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MME-VideoOCR for the released Qwen model." }, { "benchmark_id": "benchmark_mmlongbench_doc", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlongbench_doc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MMLongBench-Doc" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlongbench_doc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLongBench-Doc for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MMMU" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmstar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MMStar" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmstar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMStar for the released Qwen model." }, { "benchmark_id": "benchmark_mmsu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmsu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row MMSU" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmsu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMSU for the released Qwen model." }, { "benchmark_id": "benchmark_mmvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MMVU" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mmvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMVU for the released Qwen model." }, { "benchmark_id": "benchmark_mvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row MVBench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_mvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names MVBench for the released Qwen model." }, { "benchmark_id": "benchmark_ocrbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row OCRBench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_ocrbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCRBench for the released Qwen model." }, { "benchmark_id": "benchmark_odinw13", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row ODinW13" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_odinw13" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names ODinW13 for the released Qwen model." }, { "benchmark_id": "benchmark_omni_cloze", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_omni_cloze", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio-Visual; benchmark row Omni-Cloze" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_omni_cloze" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names Omni-Cloze for the released Qwen model." }, { "benchmark_id": "benchmark_omnigaia", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_omnigaia", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio-Visual; benchmark row OmniGAIA" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_omnigaia" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names OmniGAIA for the released Qwen model." }, { "benchmark_id": "benchmark_opencpop", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_opencpop", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row Opencpop" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_opencpop" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names Opencpop for the released Qwen model." }, { "benchmark_id": "benchmark_pmc_vqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_pmc_vqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row PMC-VQA" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_pmc_vqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names PMC-VQA for the released Qwen model." }, { "benchmark_id": "benchmark_qualcomminteractive", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_qualcomminteractive", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio-Visual; benchmark row QualcommInteractive" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_qualcomminteractive" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names QualcommInteractive for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row RealWorldQA" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_refcoco", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row RefCOCO" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_refcoco" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefCOCO for the released Qwen model." }, { "benchmark_id": "benchmark_rul_muchomusic", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_rul_muchomusic", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row RUL-MuchoMusic" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_rul_muchomusic" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names RUL-MuchoMusic for the released Qwen model." }, { "benchmark_id": "benchmark_simplevqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row SimpleVQA" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_simplevqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names SimpleVQA for the released Qwen model." }, { "benchmark_id": "benchmark_slake", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_slake", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row SLAKE" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_slake" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names SLAKE for the released Qwen model." }, { "benchmark_id": "benchmark_songformbench_chinese", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_songformbench_chinese", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row SongFormBench Chinese" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_songformbench_chinese" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names SongFormBench Chinese for the released Qwen model." }, { "benchmark_id": "benchmark_songformbench_harmonixset", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_songformbench_harmonixset", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row SongFormBench HarmonixSet" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_songformbench_harmonixset" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names SongFormBench HarmonixSet for the released Qwen model." }, { "benchmark_id": "benchmark_speechrole", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_speechrole", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row SpeechRole" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_speechrole" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names SpeechRole for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row SuperGPQA" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_tau2_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_tau2_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text; benchmark row TAU2-Bench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_tau2_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names TAU2-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_uro_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_uro_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row URO-Bench-Pro" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_uro_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names URO-Bench-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_with_audio", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_videomme_with_audio", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio-Visual; benchmark row VideoMME with audio" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_videomme_with_audio" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME with audio for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_without_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-free evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row VideoMME without subtitles" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_videomme_without_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME without subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_voicebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_voicebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row VoiceBench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_voicebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names VoiceBench for the released Qwen model." }, { "benchmark_id": "benchmark_wenetspeech", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_wenetspeech", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row WenetSpeech" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_wenetspeech" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names WenetSpeech for the released Qwen model." }, { "benchmark_id": "benchmark_wildspeech_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_wildspeech_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio; benchmark row WildSpeech-Bench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_wildspeech_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names WildSpeech-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_worldsense", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_worldsense", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Audio-Visual; benchmark row WorldSense" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_worldsense" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names WorldSense for the released Qwen model." }, { "benchmark_id": "benchmark_zerobench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_zerobench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row ZEROBench" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_zerobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZEROBench for the released Qwen model." }, { "benchmark_id": "benchmark_zerobench_sub", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Visual; benchmark row ZEROBench-Sub" }, "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260330_qwen3_5_omni_zerobench_sub" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "release_id": "release_qwen_20260330_qwen3_5_omni", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "source_revision_id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZEROBench-Sub for the released Qwen model." }, { "benchmark_id": "benchmark_aa_lcr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_aa_lcr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row AA-LCR" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_aa_lcr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names AA-LCR for the released Qwen model." }, { "benchmark_id": "benchmark_ai2d_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row AI2D Test" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ai2d_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names AI2D Test for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2026", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_aime_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row AIME 2026" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_aime_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2026 for the released Qwen model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row C-Eval" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names C-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_cc_ocr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CC-OCR" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_cc_ocr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names CC-OCR for the released Qwen model." }, { "benchmark_id": "benchmark_charxiv_rq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CharXiv RQ" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_charxiv_rq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharXiv RQ for the released Qwen model." }, { "benchmark_id": "benchmark_claweval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row ClawEval" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_claweval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names ClawEval for the released Qwen model." }, { "benchmark_id": "benchmark_countbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CountBench" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_countbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names CountBench for the released Qwen model." }, { "benchmark_id": "benchmark_deep_planning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Deep-Planning" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_deep_planning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names Deep-Planning for the released Qwen model." }, { "benchmark_id": "benchmark_dynamath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_dynamath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row DynaMath" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_dynamath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names DynaMath for the released Qwen model." }, { "benchmark_id": "benchmark_erqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ERQA" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_erqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names ERQA for the released Qwen model." }, { "benchmark_id": "benchmark_global_piqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_global_piqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Global PIQA" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_global_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names Global PIQA for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row GPQA" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HLE" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE for the released Qwen model." }, { "benchmark_id": "benchmark_hle_with_tools", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HLE with tools" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hle_with_tools" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE with tools for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_february_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT February 2025" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_february_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT February 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_february_2026", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT February 2026" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_february_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT February 2026 for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_november_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT November 2025" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_hmmt_november_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT November 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_ifbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row IFBench" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ifbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFBench for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row IFEval" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row IMOAnswerBench" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names IMOAnswerBench for the released Qwen model." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row INCLUDE" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names INCLUDE for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_longbench_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_longbench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row LongBench v2" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_longbench_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names LongBench v2 for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MathVision" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_maxife", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_maxife", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MAXIFE" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_maxife" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MAXIFE for the released Qwen model." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mcp_atlas", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MCP-Atlas" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MCP-Atlas for the released Qwen model." }, { "benchmark_id": "benchmark_mcp_mark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MCP-Mark" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mcp_mark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MCP-Mark for the released Qwen model." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MLVU" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLVU for the released Qwen model." }, { "benchmark_id": "benchmark_mmlongbench_doc", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlongbench_doc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMLongBench-Doc" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlongbench_doc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLongBench-Doc for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_prox", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMLU-ProX" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_prox" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-ProX for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMMLU" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMMU" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmstar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMStar" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_mmstar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMStar for the released Qwen model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row NL2Repo" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names NL2Repo for the released Qwen model." }, { "benchmark_id": "benchmark_nova", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_nova", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row NOVA" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_nova" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names NOVA for the released Qwen model." }, { "benchmark_id": "benchmark_odinw13", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ODinW13" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_odinw13" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names ODinW13 for the released Qwen model." }, { "benchmark_id": "benchmark_omnidocbench_1_5", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row OmniDocBench 1.5" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_omnidocbench_1_5" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names OmniDocBench 1.5 for the released Qwen model." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_osworld_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row OSWorld Verified" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names OSWorld Verified for the released Qwen model." }, { "benchmark_id": "benchmark_polymath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_polymath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row PolyMATH" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_polymath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names PolyMATH for the released Qwen model." }, { "benchmark_id": "benchmark_qwenclawbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_qwenclawbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row QwenClawBench" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_qwenclawbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenClawBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenwebbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_qwenwebbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row QwenWebBench" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_qwenwebbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenWebBench for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RealWorldQA" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_refcoco", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RefCOCO" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_refcoco" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefCOCO for the released Qwen model." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ScreenSpot Pro" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names ScreenSpot Pro for the released Qwen model." }, { "benchmark_id": "benchmark_simplevqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row SimpleVQA" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_simplevqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names SimpleVQA for the released Qwen model." }, { "benchmark_id": "benchmark_skillsbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SkillsBench" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_skillsbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names SkillsBench for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SuperGPQA" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Multilingual" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Pro" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Pro for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Verified" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Verified for the released Qwen model." }, { "benchmark_id": "benchmark_tau3_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tau3_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row TAU3-Bench" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tau3_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names TAU3-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Terminal-Bench 2.0" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names Terminal-Bench 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_tir_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tir_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row TIR-Bench" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tir_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names TIR-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_tool_decathlon", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tool_decathlon", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Tool-Decathlon" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_tool_decathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names Tool-Decathlon for the released Qwen model." }, { "benchmark_id": "benchmark_v", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_v_star", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row V*" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_v_star" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names V* for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_with_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-enabled evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMME with subtitles" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videomme_with_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME with subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_without_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-free evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMME without subtitles" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videomme_without_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME without subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videommmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMMMU" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_videommmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMMMU for the released Qwen model." }, { "benchmark_id": "benchmark_vita_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row VITA-Bench" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_vita_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names VITA-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_we_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_we_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row We-Math" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_we_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names We-Math for the released Qwen model." }, { "benchmark_id": "benchmark_widesearch", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row WideSearch" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_widesearch" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names WideSearch for the released Qwen model." }, { "benchmark_id": "benchmark_wmt24", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_wmt24_plus_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row WMT24++" }, "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260402_qwen3_6_wmt24_plus_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "release_id": "release_qwen_20260402_qwen3_6", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260402_qwen3_6", "source_revision_id": "revision_qwen_revision_qwen_20260402_qwen3_6", "source_type": "launch_page", "summary": "The official Qwen release reporting names WMT24++ for the released Qwen model." }, { "benchmark_id": "benchmark_ai2d_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_ai2d_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row AI2D Test" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_ai2d_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names AI2D Test for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2026", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_aime_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row AIME 2026" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_aime_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2026 for the released Qwen model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row C-Eval" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names C-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_cc_ocr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CC-OCR" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_cc_ocr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names CC-OCR for the released Qwen model." }, { "benchmark_id": "benchmark_charxiv_rq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CharXiv RQ" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_charxiv_rq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharXiv RQ for the released Qwen model." }, { "benchmark_id": "benchmark_claweval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row ClawEval" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_claweval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names ClawEval for the released Qwen model." }, { "benchmark_id": "benchmark_deep_planning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Deep-Planning" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_deep_planning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names Deep-Planning for the released Qwen model." }, { "benchmark_id": "benchmark_embspatialbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row EmbSpatialBench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_embspatialbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names EmbSpatialBench for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row GPQA" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_hallusionbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hallusionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row HallusionBench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hallusionbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HallusionBench for the released Qwen model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HLE" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_february_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT February 2025" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_february_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT February 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_february_2026", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT February 2026" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_february_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT February 2026 for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_november_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT November 2025" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_hmmt_november_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT November 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row IMOAnswerBench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names IMOAnswerBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_lvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row LVBench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_lvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names LVBench for the released Qwen model." }, { "benchmark_id": "benchmark_mathvista_mini", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mathvista_mini", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MathVista Mini" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mathvista_mini" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVista Mini for the released Qwen model." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mcp_atlas", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MCP-Atlas" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MCP-Atlas for the released Qwen model." }, { "benchmark_id": "benchmark_mcp_mark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MCP-Mark" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mcp_mark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MCP-Mark for the released Qwen model." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MLVU" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLVU for the released Qwen model." }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMBench EN 1.1 dev" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmbench_en_1_1_dev" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBench EN 1.1 dev for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMMU" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MVBench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_mvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MVBench for the released Qwen model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row NL2Repo" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names NL2Repo for the released Qwen model." }, { "benchmark_id": "benchmark_odinw13", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ODinW13" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_odinw13" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names ODinW13 for the released Qwen model." }, { "benchmark_id": "benchmark_omnidocbench_1_5", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row OmniDocBench 1.5" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_omnidocbench_1_5" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names OmniDocBench 1.5 for the released Qwen model." }, { "benchmark_id": "benchmark_qwenclawbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_qwenclawbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row QwenClawBench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_qwenclawbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenClawBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenwebbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_qwenwebbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row QwenWebBench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_qwenwebbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenWebBench for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RealWorldQA" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_refcoco", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RefCOCO" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_refcoco" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefCOCO for the released Qwen model." }, { "benchmark_id": "benchmark_refspatialbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_refspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RefSpatialBench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_refspatialbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefSpatialBench for the released Qwen model." }, { "benchmark_id": "benchmark_simplevqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row SimpleVQA" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_simplevqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SimpleVQA for the released Qwen model." }, { "benchmark_id": "benchmark_skillsbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SkillsBench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_skillsbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SkillsBench for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SuperGPQA" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Multilingual" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Pro" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Pro for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Verified" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Verified for the released Qwen model." }, { "benchmark_id": "benchmark_tau3_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_tau3_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row TAU3-Bench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_tau3_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names TAU3-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Terminal-Bench 2.0" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names Terminal-Bench 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_tool_decathlon", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_tool_decathlon", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Tool-Decathlon" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_tool_decathlon" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names Tool-Decathlon for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_with_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-enabled evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMME with subtitles" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videomme_with_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME with subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_without_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-free evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videomme_without_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMME without subtitles" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videomme_without_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME without subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videommmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMMMU" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_videommmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMMMU for the released Qwen model." }, { "benchmark_id": "benchmark_vita_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row VITA-Bench" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_vita_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names VITA-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_widesearch", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row WideSearch" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_widesearch" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names WideSearch for the released Qwen model." }, { "benchmark_id": "benchmark_zerobench_sub", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ZEROBench-Sub" }, "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260415_qwen3_6_35b_a3b_zerobench_sub" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "release_id": "release_qwen_20260415_qwen3_6_35b_a3b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "source_revision_id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZEROBench-Sub for the released Qwen model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Performance and Summary; benchmark label NL2Repo" }, "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names NL2Repo for the released Qwen model." }, { "benchmark_id": "benchmark_qwenchinesebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenchinesebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Performance and Summary; benchmark label QwenChineseBench" }, "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenchinesebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenChineseBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenclawbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenclawbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Performance and Summary; benchmark label QwenClawBench" }, "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenclawbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenClawBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenwebbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenwebbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Performance and Summary; benchmark label QwenWebBench" }, "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_qwenwebbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenWebBench for the released Qwen model." }, { "benchmark_id": "benchmark_scicode", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_scicode", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Performance and Summary; benchmark label SciCode" }, "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_scicode" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names SciCode for the released Qwen model." }, { "benchmark_id": "benchmark_skillsbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Performance and Summary; benchmark label SkillsBench" }, "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_skillsbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names SkillsBench for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Performance and Summary; benchmark label SuperGPQA" }, "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Performance and Summary; benchmark label SWE-bench Pro" }, "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Pro for the released Qwen model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Performance and Summary; benchmark label Terminal-Bench 2.0" }, "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names Terminal-Bench 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_toolcallformatifbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_toolcallformatifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "locator": { "kind": "figure", "value": "Figure or source text at sections Performance and Summary; benchmark label ToolcallFormatIFBench" }, "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260418_qwen3_6_max_preview_toolcallformatifbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "release_id": "release_qwen_20260418_qwen3_6_max_preview", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "source_revision_id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "source_type": "launch_page", "summary": "The official Qwen release reporting names ToolcallFormatIFBench for the released Qwen model." }, { "benchmark_id": "benchmark_aime_2026", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_aime_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row AIME 2026" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_aime_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names AIME 2026 for the released Qwen model." }, { "benchmark_id": "benchmark_androidworld", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row AndroidWorld" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_androidworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names AndroidWorld for the released Qwen model." }, { "benchmark_id": "benchmark_c_eval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_c_eval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row C-Eval" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_c_eval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names C-Eval for the released Qwen model." }, { "benchmark_id": "benchmark_cc_ocr", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_cc_ocr", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CC-OCR" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_cc_ocr" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names CC-OCR for the released Qwen model." }, { "benchmark_id": "benchmark_charxiv_rq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CharXiv RQ" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_charxiv_rq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharXiv RQ for the released Qwen model." }, { "benchmark_id": "benchmark_claweval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row ClawEval" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_claweval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names ClawEval for the released Qwen model." }, { "benchmark_id": "benchmark_countbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_countbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CountBench" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_countbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names CountBench for the released Qwen model." }, { "benchmark_id": "benchmark_dynamath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_dynamath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row DynaMath" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_dynamath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names DynaMath for the released Qwen model." }, { "benchmark_id": "benchmark_embspatialbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_embspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row EmbSpatialBench" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_embspatialbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names EmbSpatialBench for the released Qwen model." }, { "benchmark_id": "benchmark_erqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ERQA" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_erqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names ERQA for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row GPQA-Diamond" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA-Diamond for the released Qwen model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HLE" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_february_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_february_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT February 2025" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_february_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT February 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_february_2026", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT February 2026" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_february_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT February 2026 for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_november_2025", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_november_2025", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HMMT November 2025" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_hmmt_november_2025" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT November 2025 for the released Qwen model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row IMOAnswerBench" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names IMOAnswerBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_mathvista_mini", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mathvista_mini", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MathVista Mini" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mathvista_mini" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVista Mini for the released Qwen model." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MLVU" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLVU for the released Qwen model." }, { "benchmark_id": "benchmark_mmbench_en_1_1_dev", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmbench_en_1_1_dev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMBench EN 1.1 dev" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmbench_en_1_1_dev" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBench EN 1.1 dev for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMMU" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmstar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MMStar" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mmstar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMStar for the released Qwen model." }, { "benchmark_id": "benchmark_mvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MVBench" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_mvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names MVBench for the released Qwen model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row NL2Repo" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names NL2Repo for the released Qwen model." }, { "benchmark_id": "benchmark_ocrbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_ocrbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row OCRBench" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_ocrbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCRBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenclawbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_qwenclawbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row QwenClawBench" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_qwenclawbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenClawBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenwebbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_qwenwebbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row QwenWebBench" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_qwenwebbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenWebBench for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RealWorldQA" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_refcoco", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_refcoco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RefCOCO" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_refcoco" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefCOCO for the released Qwen model." }, { "benchmark_id": "benchmark_refspatialbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_refspatialbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RefSpatialBench" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_refspatialbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefSpatialBench for the released Qwen model." }, { "benchmark_id": "benchmark_simplevqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row SimpleVQA" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_simplevqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SimpleVQA for the released Qwen model." }, { "benchmark_id": "benchmark_skillsbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SkillsBench" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_skillsbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SkillsBench for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SuperGPQA" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Multilingual" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Pro" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Pro for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Verified" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Verified for the released Qwen model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Terminal-Bench 2.0" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names Terminal-Bench 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_v", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_v_star", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row V*" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_v_star" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names V* for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_with_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-enabled evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMME with subtitles" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_videomme_with_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME with subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videommmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VideoMMMU" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_videommmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMMMU for the released Qwen model." }, { "benchmark_id": "benchmark_vlmsareblind", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_vlmsareblind", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row VlmsAreBlind" }, "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260422_qwen3_6_27b_vlmsareblind" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "release_id": "release_qwen_20260422_qwen3_6_27b", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "source_revision_id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "source_type": "launch_page", "summary": "The official Qwen release reporting names VlmsAreBlind for the released Qwen model." }, { "benchmark_id": "benchmark_apex", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_apex", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row Apex" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_apex" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names Apex for the released Qwen model." }, { "benchmark_id": "benchmark_bfcl_v4", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_bfcl_v4", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row BFCL v4" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_bfcl_v4" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names BFCL v4 for the released Qwen model." }, { "benchmark_id": "benchmark_claweval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row ClawEval" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_claweval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names ClawEval for the released Qwen model." }, { "benchmark_id": "benchmark_coworkbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_coworkbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row CoWorkBench" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_coworkbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names CoWorkBench for the released Qwen model." }, { "benchmark_id": "benchmark_critpt", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_critpt", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row CritPT" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_critpt" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names CritPT for the released Qwen model." }, { "benchmark_id": "benchmark_global_piqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_global_piqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row Global PIQA" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_global_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names Global PIQA for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row GPQA-Diamond" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA-Diamond for the released Qwen model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row HLE" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE for the released Qwen model." }, { "benchmark_id": "benchmark_hle_with_tools", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row HLE with tools" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hle_with_tools" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE with tools for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_february_2026", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row HMMT February 2026" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_hmmt_february_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT February 2026 for the released Qwen model." }, { "benchmark_id": "benchmark_ifbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row IFBench" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_ifbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFBench for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row IFEval" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row IMOAnswerBench" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names IMOAnswerBench for the released Qwen model." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row INCLUDE" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names INCLUDE for the released Qwen model." }, { "benchmark_id": "benchmark_kernel_bench_l3", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_kernel_bench_l3", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row Kernel Bench L3" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_kernel_bench_l3" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names Kernel Bench L3 for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_maxife", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_maxife", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row MAXIFE" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_maxife" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names MAXIFE for the released Qwen model." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mcp_atlas", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row MCP-Atlas" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names MCP-Atlas for the released Qwen model." }, { "benchmark_id": "benchmark_mcp_mark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row MCP-Mark" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mcp_mark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names MCP-Mark for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_prox", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row MMLU-ProX" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_prox" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-ProX for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row MMMLU" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mrcr_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mrcr_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row MRCR v2" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_mrcr_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names MRCR v2 for the released Qwen model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row NL2Repo" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names NL2Repo for the released Qwen model." }, { "benchmark_id": "benchmark_nova", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_nova", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row NOVA" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_nova" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names NOVA for the released Qwen model." }, { "benchmark_id": "benchmark_polymath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_polymath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row PolyMATH" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_polymath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names PolyMATH for the released Qwen model." }, { "benchmark_id": "benchmark_qwenclaw", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenclaw", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row Qwenclaw" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenclaw" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names Qwenclaw for the released Qwen model." }, { "benchmark_id": "benchmark_qwensvg", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwensvg", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row QwenSVG" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwensvg" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenSVG for the released Qwen model." }, { "benchmark_id": "benchmark_qwenwebdev", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenwebdev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row QwenWebDev" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenwebdev" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenWebDev for the released Qwen model." }, { "benchmark_id": "benchmark_qwenworldbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenworldbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row QwenWorldBench" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_qwenworldbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenWorldBench for the released Qwen model." }, { "benchmark_id": "benchmark_scicode", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_scicode", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row SciCode" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_scicode" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names SciCode for the released Qwen model." }, { "benchmark_id": "benchmark_skillsbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row SkillsBench" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_skillsbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names SkillsBench for the released Qwen model." }, { "benchmark_id": "benchmark_spreadsheetbench_v1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_spreadsheetbench_v1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row SpreadSheetBench-v1" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_spreadsheetbench_v1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names SpreadSheetBench-v1 for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row SuperGPQA" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal agent scaffold with shell and file-edit tools" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell and file-edit tools enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row SWE-bench Multilingual" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal agent scaffold with shell and file-edit tools" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell and file-edit tools enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row SWE-bench Pro" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Pro for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal agent scaffold with shell and file-edit tools" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell and file-edit tools enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row SWE-bench Verified" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Verified for the released Qwen model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Harbor Terminus harness" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row Terminal-Bench 2.0" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names Terminal-Bench 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_vita_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row VITA-Bench" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_vita_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names VITA-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_wmt24", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_wmt24_plus_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "locator": { "kind": "table", "value": "Table under heading Performance; benchmark row WMT24++" }, "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260520_qwen3_7_wmt24_plus_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "release_id": "release_qwen_20260520_qwen3_7", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260520_qwen3_7", "source_revision_id": "revision_qwen_revision_qwen_20260520_qwen3_7", "source_type": "launch_page", "summary": "The official Qwen release reporting names WMT24++ for the released Qwen model." }, { "benchmark_id": "benchmark_androidworld", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row AndroidWorld" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_androidworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names AndroidWorld for the released Qwen model." }, { "benchmark_id": "benchmark_apex", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_apex", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row Apex" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_apex" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names Apex for the released Qwen model." }, { "benchmark_id": "benchmark_babyvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_babyvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row BabyVision" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_babyvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names BabyVision for the released Qwen model." }, { "benchmark_id": "benchmark_bc_vl", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search augmentation enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_bc_vl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row BC-VL" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_bc_vl" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names BC-VL for the released Qwen model." }, { "benchmark_id": "benchmark_bfcl_v4", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_bfcl_v4", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row BFCL v4" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_bfcl_v4" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names BFCL v4 for the released Qwen model." }, { "benchmark_id": "benchmark_charxiv_rq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row CharXiv RQ" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_charxiv_rq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharXiv RQ for the released Qwen model." }, { "benchmark_id": "benchmark_claweval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_claweval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row ClawEval" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_claweval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names ClawEval for the released Qwen model." }, { "benchmark_id": "benchmark_claweval_mm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_claweval_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row ClawEval-MM" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_claweval_mm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names ClawEval-MM for the released Qwen model." }, { "benchmark_id": "benchmark_countqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_countqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row CountQA" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_countqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names CountQA for the released Qwen model." }, { "benchmark_id": "benchmark_coworkbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_coworkbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row CoWorkBench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_coworkbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names CoWorkBench for the released Qwen model." }, { "benchmark_id": "benchmark_critpt", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_critpt", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row CritPT" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_critpt" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names CritPT for the released Qwen model." }, { "benchmark_id": "benchmark_deep_planning", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_deep_planning", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row Deep-Planning" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_deep_planning" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names Deep-Planning for the released Qwen model." }, { "benchmark_id": "benchmark_ego3d_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ego3d_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row Ego3D-Bench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ego3d_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names Ego3D-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_erqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row ERQA" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_erqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names ERQA for the released Qwen model." }, { "benchmark_id": "benchmark_global_piqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_global_piqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row Global PIQA" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_global_piqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names Global PIQA for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row GPQA-Diamond" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA-Diamond for the released Qwen model." }, { "benchmark_id": "benchmark_hipho", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hipho", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row HiPhO" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hipho" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names HiPhO for the released Qwen model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row HLE" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE for the released Qwen model." }, { "benchmark_id": "benchmark_hmmt_february_2026", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hmmt_february_2026", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row HMMT February 2026" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_hmmt_february_2026" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names HMMT February 2026 for the released Qwen model." }, { "benchmark_id": "benchmark_ifbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row IFBench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ifbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFBench for the released Qwen model." }, { "benchmark_id": "benchmark_ifeval", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ifeval", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row IFEval" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ifeval" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFEval for the released Qwen model." }, { "benchmark_id": "benchmark_imoanswerbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_imoanswerbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row IMOAnswerBench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_imoanswerbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names IMOAnswerBench for the released Qwen model." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row INCLUDE" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names INCLUDE for the released Qwen model." }, { "benchmark_id": "benchmark_kernel_bench_l3", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_kernel_bench_l3", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row Kernel Bench L3" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_kernel_bench_l3" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names Kernel Bench L3 for the released Qwen model." }, { "benchmark_id": "benchmark_lingoqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_lingoqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row LingoQA" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_lingoqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names LingoQA for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_livecodebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row LiveCodeBench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_livecodebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench for the released Qwen model." }, { "benchmark_id": "benchmark_lvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row LVBench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_lvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names LVBench for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row MathVision" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_maxife", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_maxife", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row MAXIFE" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_maxife" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MAXIFE for the released Qwen model." }, { "benchmark_id": "benchmark_mcp_atlas", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mcp_atlas", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row MCP-Atlas" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mcp_atlas" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MCP-Atlas for the released Qwen model." }, { "benchmark_id": "benchmark_mcp_mark", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mcp_mark", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row MCP-Mark" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mcp_mark" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MCP-Mark for the released Qwen model." }, { "benchmark_id": "benchmark_medxpertqa_mm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_medxpertqa_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row MedXpertQA-MM" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_medxpertqa_mm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MedXpertQA-MM for the released Qwen model." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row MLVU" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLVU for the released Qwen model." }, { "benchmark_id": "benchmark_mmbc", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search augmentation enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmbc", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row MMBC" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmbc" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMBC for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_prox", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_prox", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row MMLU-ProX" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_prox" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-ProX for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row MMMLU" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmsearchplus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search augmentation enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmsearchplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row MMSearchPlus" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mmsearchplus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMSearchPlus for the released Qwen model." }, { "benchmark_id": "benchmark_mrcr_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mrcr_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row MRCR v2" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_mrcr_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names MRCR v2 for the released Qwen model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row NL2Repo" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names NL2Repo for the released Qwen model." }, { "benchmark_id": "benchmark_nova", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_nova", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row NOVA" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_nova" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names NOVA for the released Qwen model." }, { "benchmark_id": "benchmark_ocr_bench_v2_chinese", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ocr_bench_v2_chinese", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row OCR-Bench-V2 Chinese" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ocr_bench_v2_chinese" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCR-Bench-V2 Chinese for the released Qwen model." }, { "benchmark_id": "benchmark_ocr_bench_v2_english", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ocr_bench_v2_english", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row OCR-Bench-V2 English" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_ocr_bench_v2_english" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCR-Bench-V2 English for the released Qwen model." }, { "benchmark_id": "benchmark_odinw13", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_odinw13", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row ODinW13" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_odinw13" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names ODinW13 for the released Qwen model." }, { "benchmark_id": "benchmark_omnidocbench_1_5", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row OmniDocBench 1.5" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_omnidocbench_1_5" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names OmniDocBench 1.5 for the released Qwen model." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_osworld_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row OSWorld Verified" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names OSWorld Verified for the released Qwen model." }, { "benchmark_id": "benchmark_polymath", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_polymath", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row PolyMATH" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_polymath" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names PolyMATH for the released Qwen model." }, { "benchmark_id": "benchmark_qwenclaw", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenclaw", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row Qwenclaw" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenclaw" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names Qwenclaw for the released Qwen model." }, { "benchmark_id": "benchmark_qwensvg", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwensvg", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row QwenSVG" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwensvg" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenSVG for the released Qwen model." }, { "benchmark_id": "benchmark_qwenvision2code", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenvision2code", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row QwenVision2Code" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenvision2code" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenVision2Code for the released Qwen model." }, { "benchmark_id": "benchmark_qwenwebdev", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenwebdev", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row QwenWebDev" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenwebdev" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenWebDev for the released Qwen model." }, { "benchmark_id": "benchmark_qwenworldbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenworldbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row QwenWorldBench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_qwenworldbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenWorldBench for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row RealWorldQA" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_scicode", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_scicode", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row SciCode" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_scicode" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names SciCode for the released Qwen model." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row ScreenSpot Pro" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names ScreenSpot Pro for the released Qwen model." }, { "benchmark_id": "benchmark_simplevqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search augmentation enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row SimpleVQA" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_simplevqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names SimpleVQA for the released Qwen model." }, { "benchmark_id": "benchmark_skillsbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row SkillsBench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_skillsbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names SkillsBench for the released Qwen model." }, { "benchmark_id": "benchmark_spreadsheetbench_v1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_spreadsheetbench_v1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row SpreadSheetBench-v1" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_spreadsheetbench_v1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names SpreadSheetBench-v1 for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row SuperGPQA" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_surds", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_surds", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row SURDS" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_surds" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names SURDS for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal agent scaffold with shell and file-edit tools" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell and file-edit tools enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row SWE-bench Multilingual" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal agent scaffold with shell and file-edit tools" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell and file-edit tools enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row SWE-bench Pro" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Pro for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_verified", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "internal agent scaffold with shell and file-edit tools" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "shell and file-edit tools enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row SWE-bench Verified" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_swe_bench_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Verified for the released Qwen model." }, { "benchmark_id": "benchmark_terminal_bench_2_0", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Harbor Terminus harness" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_terminal_bench_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row Terminal-Bench 2.0" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_terminal_bench_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names Terminal-Bench 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_tvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_tvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row TVBench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_tvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names TVBench for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_with_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-enabled evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row VideoMME with subtitles" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_videomme_with_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME with subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videommmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row VideoMMMU" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_videommmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMMMU for the released Qwen model." }, { "benchmark_id": "benchmark_visfactor", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_visfactor", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row VisFactor" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_visfactor" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names VisFactor for the released Qwen model." }, { "benchmark_id": "benchmark_vita_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_vita_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row VITA-Bench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_vita_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names VITA-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_vladbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_vladbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row VLADBench" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_vladbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names VLADBench for the released Qwen model." }, { "benchmark_id": "benchmark_wmt24", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_wmt24_plus_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Text Benchmarks; benchmark row WMT24++" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_wmt24_plus_plus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names WMT24++ for the released Qwen model." }, { "benchmark_id": "benchmark_worldvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "disclosed", "value": "search augmentation enabled" } }, "id": "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_worldvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Multimodal Benchmarks; benchmark row WorldVQA" }, "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260601_qwen3_7_plus_worldvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "release_id": "release_qwen_20260601_qwen3_7_plus", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "source_revision_id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "source_type": "launch_page", "summary": "The official Qwen release reporting names WorldVQA for the released Qwen model." }, { "benchmark_id": "benchmark_agents_last_exam", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_agents_last_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row Agents' Last Exam" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_agents_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names Agents' Last Exam for the released Qwen model." }, { "benchmark_id": "benchmark_androidbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_androidbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row AndroidBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_androidbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names AndroidBench for the released Qwen model." }, { "benchmark_id": "benchmark_androidworld", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row AndroidWorld" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_androidworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names AndroidWorld for the released Qwen model." }, { "benchmark_id": "benchmark_automation_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_automation_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row Automation-Bench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_automation_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names Automation-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_babyvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_babyvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row BabyVision" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_babyvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names BabyVision for the released Qwen model." }, { "benchmark_id": "benchmark_cc_ocr_bench_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_cc_ocr_bench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row CC-OCR-Bench-V2" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_cc_ocr_bench_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names CC-OCR-Bench-V2 for the released Qwen model." }, { "benchmark_id": "benchmark_charxiv_rq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row CharXiv RQ" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_charxiv_rq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharXiv RQ for the released Qwen model." }, { "benchmark_id": "benchmark_claweval_mm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_claweval_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row ClawEval-MM" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_claweval_mm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names ClawEval-MM for the released Qwen model." }, { "benchmark_id": "benchmark_coco", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_coco", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row COCO" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_coco" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names COCO for the released Qwen model." }, { "benchmark_id": "benchmark_countqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_countqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row CountQA" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_countqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names CountQA for the released Qwen model." }, { "benchmark_id": "benchmark_coworkbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_coworkbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row CoWorkBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_coworkbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names CoWorkBench for the released Qwen model." }, { "benchmark_id": "benchmark_deepswe_1_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_deepswe_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row DeepSWE 1.1" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_deepswe_1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names DeepSWE 1.1 for the released Qwen model." }, { "benchmark_id": "benchmark_dense200", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_dense200", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row Dense200" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_dense200" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names Dense200 for the released Qwen model." }, { "benchmark_id": "benchmark_egolife_w_mem", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_egolife_w_mem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row EgoLife (w/ Mem.)" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_egolife_w_mem" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names EgoLife (w/ Mem.) for the released Qwen model." }, { "benchmark_id": "benchmark_erqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row ERQA" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_erqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names ERQA for the released Qwen model." }, { "benchmark_id": "benchmark_frontierswe", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_frontierswe", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row FrontierSWE" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_frontierswe" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names FrontierSWE for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row GPQA-Diamond" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA-Diamond for the released Qwen model." }, { "benchmark_id": "benchmark_healthbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_healthbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row HealthBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_healthbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names HealthBench for the released Qwen model." }, { "benchmark_id": "benchmark_hipho", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hipho", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row HiPhO" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hipho" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names HiPhO for the released Qwen model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row HLE" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE for the released Qwen model." }, { "benchmark_id": "benchmark_hle_vl_w_tools", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle_vl_w_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row HLE-VL (w/ Tools)" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle_vl_w_tools" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE-VL (w/ Tools) for the released Qwen model." }, { "benchmark_id": "benchmark_hle_with_tools", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle_with_tools", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row HLE with tools" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_hle_with_tools" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE with tools for the released Qwen model." }, { "benchmark_id": "benchmark_ifbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row IFBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_ifbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFBench for the released Qwen model." }, { "benchmark_id": "benchmark_jobbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_jobbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row JobBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_jobbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names JobBench for the released Qwen model." }, { "benchmark_id": "benchmark_lingoqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lingoqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row LingoQA" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lingoqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names LingoQA for the released Qwen model." }, { "benchmark_id": "benchmark_logicvista", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_logicvista", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row LogicVista" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_logicvista" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names LogicVista for the released Qwen model." }, { "benchmark_id": "benchmark_longbench_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_longbench_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row LongBench v2" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_longbench_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names LongBench v2 for the released Qwen model." }, { "benchmark_id": "benchmark_lvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row LVBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names LVBench for the released Qwen model." }, { "benchmark_id": "benchmark_lvbench_w_mem", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lvbench_w_mem", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row LVBench (w/ Mem.)" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_lvbench_w_mem" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names LVBench (w/ Mem.) for the released Qwen model." }, { "benchmark_id": "benchmark_madqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_madqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MADQA" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_madqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MADQA for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MathVision" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_medxpertqa_mm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_medxpertqa_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MedXpertQA-MM" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_medxpertqa_mm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MedXpertQA-MM for the released Qwen model." }, { "benchmark_id": "benchmark_mls_bench_lite", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mls_bench_lite", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MLS-Bench-Lite" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mls_bench_lite" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLS-Bench-Lite for the released Qwen model." }, { "benchmark_id": "benchmark_mlvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mlvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MLVU" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mlvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MLVU for the released Qwen model." }, { "benchmark_id": "benchmark_mmmu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmmu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MMMU-Pro" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmmu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmstar", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmstar", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MMStar" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmstar" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMStar for the released Qwen model." }, { "benchmark_id": "benchmark_mmvu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmvu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MMVU" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mmvu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMVU for the released Qwen model." }, { "benchmark_id": "benchmark_mobileworld", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mobileworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MobileWorld" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mobileworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MobileWorld for the released Qwen model." }, { "benchmark_id": "benchmark_mrcr_v2", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mrcr_v2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MRCR v2" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mrcr_v2" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MRCR v2 for the released Qwen model." }, { "benchmark_id": "benchmark_mtvqa_test", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mtvqa_test", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row MTVQA-Test" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_mtvqa_test" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names MTVQA-Test for the released Qwen model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row NL2Repo" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names NL2Repo for the released Qwen model." }, { "benchmark_id": "benchmark_ocr_bench_v2_en_zh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_ocr_bench_v2_en_zh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row OCR-Bench-V2 (EN/ZH)" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_ocr_bench_v2_en_zh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names OCR-Bench-V2 (EN/ZH) for the released Qwen model." }, { "benchmark_id": "benchmark_omnidocbench_1_5", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_omnidocbench_1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row OmniDocBench 1.5" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_omnidocbench_1_5" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names OmniDocBench 1.5 for the released Qwen model." }, { "benchmark_id": "benchmark_onemillion_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_onemillion_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row $OneMillion-Bench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_onemillion_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names $OneMillion-Bench for the released Qwen model." }, { "benchmark_id": "benchmark_osworld_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_osworld_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row OSWorld 2.0" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_osworld_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names OSWorld 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_osworld_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_osworld_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row OSWorld Verified" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_osworld_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names OSWorld Verified for the released Qwen model." }, { "benchmark_id": "benchmark_paperbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_paperbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row PaperBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_paperbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names PaperBench for the released Qwen model." }, { "benchmark_id": "benchmark_parametric_cad_bench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_parametric_cad_bench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row Parametric CAD Bench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_parametric_cad_bench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names Parametric CAD Bench for the released Qwen model." }, { "benchmark_id": "benchmark_perceptionbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_perceptionbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row PerceptionBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_perceptionbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names PerceptionBench for the released Qwen model." }, { "benchmark_id": "benchmark_phyx", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_phyx", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row PhyX" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_phyx" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names PhyX for the released Qwen model." }, { "benchmark_id": "benchmark_plawbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_plawbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row PLawBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_plawbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names PLawBench for the released Qwen model." }, { "benchmark_id": "benchmark_pmc_vqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_pmc_vqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row PMC-VQA" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_pmc_vqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names PMC-VQA for the released Qwen model." }, { "benchmark_id": "benchmark_prbench_finance", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_prbench_finance", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row PRBench-Finance" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_prbench_finance" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names PRBench-Finance for the released Qwen model." }, { "benchmark_id": "benchmark_prbench_legal", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_prbench_legal", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row PRBench-Legal" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_prbench_legal" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names PRBench-Legal for the released Qwen model." }, { "benchmark_id": "benchmark_presentbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_presentbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row PresentBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_presentbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names PresentBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenblenderbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenblenderbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row QwenBlenderBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenblenderbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenBlenderBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenqoderbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenqoderbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row QwenQoderBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenqoderbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenQoderBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenreactbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenreactbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row QwenReactBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenreactbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenReactBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwensvgbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwensvgbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row QwenSVGBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwensvgbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenSVGBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenswebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenswebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row QwenSWEBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenswebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenSWEBench for the released Qwen model." }, { "benchmark_id": "benchmark_qwenvisualoffice", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenvisualoffice", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row QwenVisualOffice" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_qwenvisualoffice" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names QwenVisualOffice for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row RealWorldQA" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_recreationbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_recreationbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row RecreationBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_recreationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names RecreationBench for the released Qwen model." }, { "benchmark_id": "benchmark_refadv_s", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_refadv_s", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row RefAdv-S" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_refadv_s" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names RefAdv-S for the released Qwen model." }, { "benchmark_id": "benchmark_screenspot_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_screenspot_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row ScreenSpot Pro" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_screenspot_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names ScreenSpot Pro for the released Qwen model." }, { "benchmark_id": "benchmark_simplevqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_simplevqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row SimpleVQA" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_simplevqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names SimpleVQA for the released Qwen model." }, { "benchmark_id": "benchmark_skillsbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_skillsbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row SkillsBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_skillsbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names SkillsBench for the released Qwen model." }, { "benchmark_id": "benchmark_slake", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_slake", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row SLAKE" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_slake" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names SLAKE for the released Qwen model." }, { "benchmark_id": "benchmark_surds", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_surds", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row SURDS" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_surds" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names SURDS for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row SWE-bench Pro" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Pro for the released Qwen model." }, { "benchmark_id": "benchmark_terminal_bench_2_1", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_terminal_bench_2_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row Terminal Bench 2.1" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_terminal_bench_2_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names Terminal Bench 2.1 for the released Qwen model." }, { "benchmark_id": "benchmark_toolathlon_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_toolathlon_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row Toolathlon Verified" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_toolathlon_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names Toolathlon Verified for the released Qwen model." }, { "benchmark_id": "benchmark_tvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_tvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row TVBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_tvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names TVBench for the released Qwen model." }, { "benchmark_id": "benchmark_videodr_w_search", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videodr_w_search", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row VideoDR (w/ Search)" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videodr_w_search" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoDR (w/ Search) for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_v2_w_sub", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videomme_v2_w_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row VideoMME v2 (w/ Sub.)" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videomme_v2_w_sub" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME v2 (w/ Sub.) for the released Qwen model." }, { "benchmark_id": "benchmark_videomme_with_subtitles", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "disclosed", "value": "subtitle-enabled evaluation variant" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videomme_with_subtitles", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row VideoMME with subtitles" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videomme_with_subtitles" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMME with subtitles for the released Qwen model." }, { "benchmark_id": "benchmark_videommmu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videommmu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row VideoMMMU" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_videommmu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names VideoMMMU for the released Qwen model." }, { "benchmark_id": "benchmark_visfactor", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_visfactor", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row VisFactor" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_visfactor" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names VisFactor for the released Qwen model." }, { "benchmark_id": "benchmark_vision2web", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_vision2web", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row Vision2Web" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_vision2web" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names Vision2Web for the released Qwen model." }, { "benchmark_id": "benchmark_vlmsarebiased", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_vlmsarebiased", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row VLMsAreBiased" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_vlmsarebiased" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names VLMsAreBiased for the released Qwen model." }, { "benchmark_id": "benchmark_webarena_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_webarena_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row WebArena-Verified" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_webarena_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names WebArena-Verified for the released Qwen model." }, { "benchmark_id": "benchmark_widesearch", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_widesearch", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row WideSearch" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_widesearch" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names WideSearch for the released Qwen model." }, { "benchmark_id": "benchmark_workspacebench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_workspacebench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row WorkSpaceBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_workspacebench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names WorkSpaceBench for the released Qwen model." }, { "benchmark_id": "benchmark_worldvqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_worldvqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row WorldVQA" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_worldvqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names WorldVQA for the released Qwen model." }, { "benchmark_id": "benchmark_zerobench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_zerobench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row ZEROBench" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_zerobench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZEROBench for the released Qwen model." }, { "benchmark_id": "benchmark_zerobench_sub", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_zerobench_sub", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "locator": { "kind": "table", "value": "Table under heading Full Benchmark Table; benchmark row ZEROBench-Sub" }, "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260803_qwen3_8_zerobench_sub" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "release_id": "release_qwen_20260803_qwen3_8", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260803_qwen3_8", "source_revision_id": "revision_qwen_revision_qwen_20260803_qwen3_8", "source_type": "launch_page", "summary": "The official Qwen release reporting names ZEROBench-Sub for the released Qwen model." }, { "benchmark_id": "benchmark_agents_last_exam", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_agents_last_exam", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Agents' Last Exam" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_agents_last_exam" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names Agents' Last Exam for the released Qwen model." }, { "benchmark_id": "benchmark_androidworld", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_androidworld", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row AndroidWorld" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_androidworld" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names AndroidWorld for the released Qwen model." }, { "benchmark_id": "benchmark_bbh", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_bbh", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row BBH" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_bbh" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names BBH for the released Qwen model." }, { "benchmark_id": "benchmark_charxiv_rq", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_charxiv_rq", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row CharXiv RQ" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_charxiv_rq" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names CharXiv RQ for the released Qwen model." }, { "benchmark_id": "benchmark_claweval_mm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_claweval_mm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ClawEval-MM" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_claweval_mm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names ClawEval-MM for the released Qwen model." }, { "benchmark_id": "benchmark_coworkbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_coworkbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row CoWorkBench" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_coworkbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names CoWorkBench for the released Qwen model." }, { "benchmark_id": "benchmark_deepswe_1_1", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "two named software-agent harnesses" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_deepswe_1_1", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row DeepSWE 1.1" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_deepswe_1_1" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names DeepSWE 1.1 for the released Qwen model." }, { "benchmark_id": "benchmark_erqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_erqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row ERQA" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_erqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names ERQA for the released Qwen model." }, { "benchmark_id": "benchmark_evalplus", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_evalplus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row EvalPlus" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_evalplus" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names EvalPlus for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row GPQA" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA for the released Qwen model." }, { "benchmark_id": "benchmark_gpqa_diamond", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gpqa_diamond", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row GPQA-Diamond" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gpqa_diamond" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names GPQA-Diamond for the released Qwen model." }, { "benchmark_id": "benchmark_gsm8k", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gsm8k", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row GSM8K" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_gsm8k" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names GSM8K for the released Qwen model." }, { "benchmark_id": "benchmark_hle", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_hle", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row HLE" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_hle" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names HLE for the released Qwen model." }, { "benchmark_id": "benchmark_ifbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_ifbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row IFBench" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_ifbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names IFBench for the released Qwen model." }, { "benchmark_id": "benchmark_include", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_include", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row INCLUDE" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_include" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names INCLUDE for the released Qwen model." }, { "benchmark_id": "benchmark_jobbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_jobbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row JobBench" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_jobbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names JobBench for the released Qwen model." }, { "benchmark_id": "benchmark_livecodebench_v6", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_livecodebench_v6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row LiveCodeBench v6" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_livecodebench_v6" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names LiveCodeBench v6 for the released Qwen model." }, { "benchmark_id": "benchmark_lvbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_lvbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row LVBench" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_lvbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names LVBench for the released Qwen model." }, { "benchmark_id": "benchmark_math", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row MATH" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_math" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MATH for the released Qwen model." }, { "benchmark_id": "benchmark_mathvision", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mathvision", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row MathVision" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mathvision" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MathVision for the released Qwen model." }, { "benchmark_id": "benchmark_mgsm", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mgsm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row MGSM" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mgsm" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MGSM for the released Qwen model." }, { "benchmark_id": "benchmark_massive_multitask_language_understanding", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row MMLU" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_pro", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row MMLU-Pro" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Pro for the released Qwen model." }, { "benchmark_id": "benchmark_mmlu_redux", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu_redux", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row MMLU-Redux" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmlu_redux" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMLU-Redux for the released Qwen model." }, { "benchmark_id": "benchmark_mmmlu", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmmlu", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row MMMLU" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_mmmlu" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MMMLU for the released Qwen model." }, { "benchmark_id": "benchmark_multipl_e", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_multipl_e", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row MultiPL-E" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_multipl_e" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names MultiPL-E for the released Qwen model." }, { "benchmark_id": "benchmark_nl2repo", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Claude Code harness" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_nl2repo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row NL2Repo" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_nl2repo" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names NL2Repo for the released Qwen model." }, { "benchmark_id": "benchmark_osworld_2_0", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_osworld_2_0", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row OSWorld 2.0" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_osworld_2_0" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names OSWorld 2.0 for the released Qwen model." }, { "benchmark_id": "benchmark_realworldqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_realworldqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RealWorldQA" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_realworldqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names RealWorldQA for the released Qwen model." }, { "benchmark_id": "benchmark_recreationbench", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_recreationbench", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row RecreationBench" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_recreationbench" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names RecreationBench for the released Qwen model." }, { "benchmark_id": "benchmark_supergpqa", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_supergpqa", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row SuperGPQA" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_supergpqa" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names SuperGPQA for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_multilingual", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swe_bench_multilingual", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Multilingual" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swe_bench_multilingual" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Multilingual for the released Qwen model." }, { "benchmark_id": "benchmark_swe_bench_pro", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Claude Code harness" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swe_bench_pro", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row SWE-bench Pro" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swe_bench_pro" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWE-bench Pro for the released Qwen model." }, { "benchmark_id": "benchmark_swebench_pretrain", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swebench_pretrain", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Base Model Performance; benchmark row SWEBench-Pretrain" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_swebench_pretrain" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names SWEBench-Pretrain for the released Qwen model." }, { "benchmark_id": "benchmark_toolathlon_verified", "evaluation_setup": { "evaluation_harness": { "status": "not_disclosed" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_toolathlon_verified", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Language; benchmark row Toolathlon Verified" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_toolathlon_verified" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names Toolathlon Verified for the released Qwen model." }, { "benchmark_id": "benchmark_vision2web", "evaluation_setup": { "evaluation_harness": { "status": "disclosed", "value": "Claude Code harness with named model judge" }, "prompting": { "status": "not_disclosed" }, "sample_selection": { "status": "not_disclosed" }, "shot_configuration": { "status": "not_disclosed" }, "tool_use": { "status": "not_disclosed" } }, "id": "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_vision2web", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "locator": { "kind": "table", "value": "Table under heading Vision Language; benchmark row Vision2Web" }, "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "provenance": { "record_ids": [ "occurrence_qwen_occurrence_qwen_20260826_qwen3_8_flash_next_vision2web" ], "record_type": "occurrences", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "release_id": "release_qwen_20260826_qwen3_8_flash_next", "review_status": "verified", "source_id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "source_revision_id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "source_type": "launch_page", "summary": "The official Qwen release reporting names Vision2Web for the released Qwen model." } ], "quarantine": [ { "candidate_id": "candidate_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer", "evidence_gap": "Public material does not identify the model precisely enough for a verified release identity.", "id": "quarantine_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer", "lab_id": "anthropic", "provenance": { "record_ids": [ "quarantine_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer" ], "record_type": "quarantine", "source_file": "data/catalog.yaml" }, "reason_code": "insufficient_public_model_identity", "source_id": "source_candidate_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer" } ], "release_candidates": [ { "disposition": "included", "id": "candidate_anthropic_20240304_introducing_the_next_generation_of_claude", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "source_id": "source_candidate_anthropic_20240304_introducing_the_next_generation_of_claude", "title": "Introducing the next generation of Claude" }, { "canonical_candidate_id": "candidate_anthropic_20240304_introducing_the_next_generation_of_claude", "disposition": "duplicate_or_alias", "id": "candidate_anthropic_20240313_claude_3_haiku_our_fastest_model_yet", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20240313_claude_3_haiku_our_fastest_model_yet" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-13", "source_id": "source_candidate_anthropic_20240313_claude_3_haiku_our_fastest_model_yet", "title": "Claude 3 Haiku: our fastest model yet" }, { "disposition": "included", "id": "candidate_anthropic_20240621_claude_3_5_sonnet", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "source_id": "source_candidate_anthropic_20240621_claude_3_5_sonnet", "title": "Claude 3.5 Sonnet" }, { "disposition": "included", "id": "candidate_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "source_id": "source_candidate_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "title": "Introducing computer use, a new Claude 3.5 Sonnet, and Claude 3.5 Haiku" }, { "disposition": "included", "id": "candidate_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "source_id": "source_candidate_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "title": "Claude 3.7 Sonnet and Claude Code" }, { "disposition": "included", "id": "candidate_anthropic_20250522_introducing_claude_4", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20250522_introducing_claude_4" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "source_id": "source_candidate_anthropic_20250522_introducing_claude_4", "title": "Introducing Claude 4" }, { "disposition": "quarantined", "id": "candidate_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-06", "reason_code": "insufficient_public_model_identity", "source_id": "source_candidate_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer", "title": "Claude Gov models for U.S. national security customers" }, { "disposition": "included", "id": "candidate_anthropic_20250805_claude_opus_4_1", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20250805_claude_opus_4_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "source_id": "source_candidate_anthropic_20250805_claude_opus_4_1", "title": "Claude Opus 4.1" }, { "disposition": "included", "id": "candidate_anthropic_20250929_introducing_claude_sonnet_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "source_id": "source_candidate_anthropic_20250929_introducing_claude_sonnet_4_5", "title": "Introducing Claude Sonnet 4.5" }, { "disposition": "included", "id": "candidate_anthropic_20251015_introducing_claude_haiku_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "source_id": "source_candidate_anthropic_20251015_introducing_claude_haiku_4_5", "title": "Introducing Claude Haiku 4.5" }, { "disposition": "included", "id": "candidate_anthropic_20251124_introducing_claude_opus_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "source_id": "source_candidate_anthropic_20251124_introducing_claude_opus_4_5", "title": "Introducing Claude Opus 4.5" }, { "disposition": "included", "id": "candidate_anthropic_20260205_introducing_claude_opus_4_6", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "source_id": "source_candidate_anthropic_20260205_introducing_claude_opus_4_6", "title": "Introducing Claude Opus 4.6" }, { "disposition": "included", "id": "candidate_anthropic_20260217_introducing_claude_sonnet_4_6", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "source_id": "source_candidate_anthropic_20260217_introducing_claude_sonnet_4_6", "title": "Introducing Claude Sonnet 4.6" }, { "disposition": "included", "id": "candidate_anthropic_20260416_introducing_claude_opus_4_7", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "source_id": "source_candidate_anthropic_20260416_introducing_claude_opus_4_7", "title": "Introducing Claude Opus 4.7" }, { "disposition": "included", "id": "candidate_anthropic_20260528_introducing_claude_opus_4_8", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "source_id": "source_candidate_anthropic_20260528_introducing_claude_opus_4_8", "title": "Introducing Claude Opus 4.8" }, { "disposition": "included", "id": "candidate_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "source_id": "source_candidate_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "title": "Claude Fable 5 and Claude Mythos 5" }, { "disposition": "included", "id": "candidate_anthropic_20260630_introducing_claude_sonnet_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "source_id": "source_candidate_anthropic_20260630_introducing_claude_sonnet_5", "title": "Introducing Claude Sonnet 5" }, { "canonical_candidate_id": "candidate_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "disposition": "duplicate_or_alias", "id": "candidate_anthropic_20260630_redeploying_fable_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20260630_redeploying_fable_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "source_id": "source_candidate_anthropic_20260630_redeploying_fable_5", "title": "Redeploying Fable 5" }, { "disposition": "included", "id": "candidate_anthropic_20260724_introducing_claude_opus_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "candidate_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "source_id": "source_candidate_anthropic_20260724_introducing_claude_opus_5", "title": "Introducing Claude Opus 5" }, { "disposition": "included", "id": "candidate_deepseek_20240517_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20240517_deepseek_chat" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-17", "source_id": "source_candidate_deepseek_20240517_deepseek_chat", "title": "deepseek-chat" }, { "disposition": "included", "id": "candidate_deepseek_20240614_deepseek_coder", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20240614_deepseek_coder" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-14", "source_id": "source_candidate_deepseek_20240614_deepseek_coder", "title": "deepseek-coder" }, { "disposition": "included", "id": "candidate_deepseek_20240628_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20240628_deepseek_chat" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "source_id": "source_candidate_deepseek_20240628_deepseek_chat", "title": "deepseek-chat" }, { "disposition": "included", "id": "candidate_deepseek_20240724_deepseek_coder", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20240724_deepseek_coder" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-24", "source_id": "source_candidate_deepseek_20240724_deepseek_coder", "title": "deepseek-coder" }, { "disposition": "excluded", "id": "candidate_deepseek_20240725_new_api_features", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20240725_new_api_features" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-25", "reason_code": "not_release_announcement", "source_id": "source_candidate_deepseek_20240725_new_api_features", "title": "New API Features" }, { "disposition": "excluded", "id": "candidate_deepseek_20240802_api_launches_context_caching_on_disk_technology", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20240802_api_launches_context_caching_on_disk_technology" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-02", "reason_code": "not_release_announcement", "source_id": "source_candidate_deepseek_20240802_api_launches_context_caching_on_disk_technology", "title": "API Launches Context Caching on Disk Technology" }, { "disposition": "included", "id": "candidate_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "source_id": "source_candidate_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "title": "deepseek-coder & deepseek-chat Upgraded to DeepSeek V2.5 Model" }, { "disposition": "included", "id": "candidate_deepseek_20241120_deepseek_r1_lite_preview", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20241120_deepseek_r1_lite_preview" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-20", "source_id": "source_candidate_deepseek_20241120_deepseek_r1_lite_preview", "title": "DeepSeek-R1-Lite-Preview is now live: unleashing supercharged reasoning power!" }, { "disposition": "included", "id": "candidate_deepseek_20241210_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20241210_deepseek_chat" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "source_id": "source_candidate_deepseek_20241210_deepseek_chat", "title": "deepseek-chat" }, { "disposition": "included", "id": "candidate_deepseek_20241226_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20241226_deepseek_chat" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "source_id": "source_candidate_deepseek_20241226_deepseek_chat", "title": "deepseek-chat" }, { "disposition": "included", "id": "candidate_deepseek_20250120_deepseek_reasoner", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20250120_deepseek_reasoner" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "source_id": "source_candidate_deepseek_20250120_deepseek_reasoner", "title": "deepseek-reasoner" }, { "disposition": "included", "id": "candidate_deepseek_20250324_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20250324_deepseek_chat" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "source_id": "source_candidate_deepseek_20250324_deepseek_chat", "title": "deepseek-chat" }, { "disposition": "included", "id": "candidate_deepseek_20250528_deepseek_reasoner", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20250528_deepseek_reasoner" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "source_id": "source_candidate_deepseek_20250528_deepseek_reasoner", "title": "deepseek-reasoner" }, { "disposition": "included", "id": "candidate_deepseek_20250821_deepseek_v3_1", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20250821_deepseek_v3_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "source_id": "source_candidate_deepseek_20250821_deepseek_v3_1", "title": "DeepSeek-V3.1" }, { "disposition": "included", "id": "candidate_deepseek_20250922_deepseek_v3_1_terminus", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "source_id": "source_candidate_deepseek_20250922_deepseek_v3_1_terminus", "title": "DeepSeek-V3.1-Terminus" }, { "disposition": "included", "id": "candidate_deepseek_20250929_deepseek_v3_2_exp", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "source_id": "source_candidate_deepseek_20250929_deepseek_v3_2_exp", "title": "DeepSeek-V3.2-Exp" }, { "disposition": "included", "id": "candidate_deepseek_20251201_deepseek_v3_2", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20251201_deepseek_v3_2" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "source_id": "source_candidate_deepseek_20251201_deepseek_v3_2", "title": "DeepSeek-V3.2" }, { "disposition": "included", "id": "candidate_deepseek_20251201_deepseek_v3_2_speciale", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "source_id": "source_candidate_deepseek_20251201_deepseek_v3_2_speciale", "title": "DeepSeek-V3.2-Speciale" }, { "disposition": "included", "id": "candidate_deepseek_20260424_deepseek_v4", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20260424_deepseek_v4" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "source_id": "source_candidate_deepseek_20260424_deepseek_v4", "title": "DeepSeek-V4" }, { "disposition": "included", "id": "candidate_deepseek_20260731_deepseek_v4_flash_update", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "source_id": "source_candidate_deepseek_20260731_deepseek_v4_flash_update", "title": "DeepSeek-V4-Flash Update" }, { "disposition": "included", "id": "candidate_deepseek_20260813_deepseek_v4_pro_update", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "source_id": "source_candidate_deepseek_20260813_deepseek_v4_pro_update", "title": "DeepSeek-V4-Pro Update" }, { "disposition": "included", "id": "candidate_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "lab_id": "deepseek", "provenance": { "record_ids": [ "candidate_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "source_id": "source_candidate_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "title": "DeepSeek-V4-Flash-Vision-Exp Release" }, { "disposition": "included", "id": "candidate_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20240208_the_next_chapter_of_our_gemini_era" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-08", "source_id": "source_candidate_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "title": "The next chapter of our Gemini era" }, { "disposition": "included", "id": "candidate_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20240215_our_next_generation_model_gemini_1_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "source_id": "source_candidate_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "title": "Our next-generation model: Gemini 1.5" }, { "disposition": "included", "id": "candidate_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "source_id": "source_candidate_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "title": "Gemma: Introducing new state-of-the-art open models" }, { "disposition": "included", "id": "candidate_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "source_id": "source_candidate_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "title": "Gemini breaks new ground: a faster model, longer context and AI agents" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20240731_gemma_scope_helping_the_safety_community_shed_light", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20240731_gemma_scope_helping_the_safety_community_shed_light" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "reason_code": "not_release_announcement", "source_id": "source_candidate_google_deepmind_20240731_gemma_scope_helping_the_safety_community_shed_light", "title": "Gemma Scope: helping the safety community shed light on the inner workings of language models" }, { "disposition": "included", "id": "candidate_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "source_id": "source_candidate_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "title": "Updated production-ready Gemini models, reduced 1.5 Pro pricing, increased rate limits, and more" }, { "disposition": "included", "id": "candidate_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "source_id": "source_candidate_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "title": "Introducing Gemini 2.0: our new AI model for the agentic era" }, { "disposition": "included", "id": "candidate_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "source_id": "source_candidate_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "title": "Gemini 2.0 is now available to everyone" }, { "disposition": "included", "id": "candidate_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "source_id": "source_candidate_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "title": "Start building with Gemini 2.0 Flash and Flash-Lite" }, { "disposition": "included", "id": "candidate_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "source_id": "source_candidate_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "title": "Experiment with Gemini 2.0 Flash native image generation" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250312_gemini_robotics_brings_ai_into_the_physical_world", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250312_gemini_robotics_brings_ai_into_the_physical_world" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "reason_code": "excluded_robotics", "source_id": "source_candidate_google_deepmind_20250312_gemini_robotics_brings_ai_into_the_physical_world", "title": "Gemini Robotics brings AI into the physical world" }, { "disposition": "included", "id": "candidate_google_deepmind_20250312_introducing_gemma_3", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "source_id": "source_candidate_google_deepmind_20250312_introducing_gemma_3", "title": "Introducing Gemma 3" }, { "disposition": "included", "id": "candidate_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "source_id": "source_candidate_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "title": "Gemini 2.5: Our most intelligent AI model" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250414_dolphingemma_how_google_ai_is_helping_decode_dolphin", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250414_dolphingemma_how_google_ai_is_helping_decode_dolphin" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "reason_code": "excluded_audio_only", "source_id": "source_candidate_google_deepmind_20250414_dolphingemma_how_google_ai_is_helping_decode_dolphin", "title": "DolphinGemma: How Google AI is helping decode dolphin communication" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250415_generate_videos_in_gemini_and_whisk_with_veo_2", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250415_generate_videos_in_gemini_and_whisk_with_veo_2" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-15", "reason_code": "excluded_image_or_video_only", "source_id": "source_candidate_google_deepmind_20250415_generate_videos_in_gemini_and_whisk_with_veo_2", "title": "Generate videos in Gemini and Whisk with Veo 2" }, { "disposition": "included", "id": "candidate_google_deepmind_20250417_introducing_gemini_2_5_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "source_id": "source_candidate_google_deepmind_20250417_introducing_gemini_2_5_flash", "title": "Introducing Gemini 2.5 Flash" }, { "disposition": "included", "id": "candidate_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "source_id": "source_candidate_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "title": "Build rich, interactive web apps with an updated Gemini 2.5 Pro" }, { "disposition": "included", "id": "candidate_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "source_id": "source_candidate_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "title": "Gemini 2.5 Pro Preview: even better coding performance" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250514_alphaevolve_a_gemini_powered_coding_agent_for_design", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250514_alphaevolve_a_gemini_powered_coding_agent_for_design" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-14", "reason_code": "not_release_announcement", "source_id": "source_candidate_google_deepmind_20250514_alphaevolve_a_gemini_powered_coding_agent_for_design", "title": "AlphaEvolve: A Gemini-powered coding agent for designing advanced algorithms" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250520_advancing_gemini_s_security_safeguards", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250520_advancing_gemini_s_security_safeguards" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "reason_code": "not_release_announcement", "source_id": "source_candidate_google_deepmind_20250520_advancing_gemini_s_security_safeguards", "title": "Advancing Gemini's security safeguards" }, { "disposition": "included", "id": "candidate_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "source_id": "source_candidate_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "title": "Announcing Gemma 3n preview: Powerful, efficient, mobile-first AI" }, { "disposition": "included", "id": "candidate_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "source_id": "source_candidate_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "title": "Gemini 2.5: Our most intelligent models are getting even better" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250603_advanced_audio_dialog_and_generation_with_gemini_2_5", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250603_advanced_audio_dialog_and_generation_with_gemini_2_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-03", "reason_code": "excluded_audio_only", "source_id": "source_candidate_google_deepmind_20250603_advanced_audio_dialog_and_generation_with_gemini_2_5", "title": "Advanced audio dialog and generation with Gemini 2.5" }, { "disposition": "included", "id": "candidate_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "source_id": "source_candidate_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "title": "Gemini 2.5: Updates to our family of thinking models" }, { "canonical_candidate_id": "candidate_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "disposition": "duplicate_or_alias", "id": "candidate_google_deepmind_20250617_we_re_expanding_our_gemini_2_5_family_of_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250617_we_re_expanding_our_gemini_2_5_family_of_models" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "source_id": "source_candidate_google_deepmind_20250617_we_re_expanding_our_gemini_2_5_family_of_models", "title": "We’re expanding our Gemini 2.5 family of models" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250624_gemini_robotics_on_device_brings_ai_to_local_robotic", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250624_gemini_robotics_on_device_brings_ai_to_local_robotic" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-24", "reason_code": "excluded_robotics", "source_id": "source_candidate_google_deepmind_20250624_gemini_robotics_on_device_brings_ai_to_local_robotic", "title": "Gemini Robotics On-Device brings AI to local robotic devices" }, { "disposition": "included", "id": "candidate_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "source_id": "source_candidate_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "title": "Introducing Gemma 3n: The developer guide" }, { "disposition": "included", "id": "candidate_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "source_id": "source_candidate_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "title": "MedGemma: Our most capable open models for health AI development" }, { "disposition": "included", "id": "candidate_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "source_id": "source_candidate_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "title": "T5Gemma: A new collection of encoder-decoder Gemma models" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250721_advanced_version_of_gemini_with_deep_think_officiall", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250721_advanced_version_of_gemini_with_deep_think_officiall" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-21", "reason_code": "not_release_announcement", "source_id": "source_candidate_google_deepmind_20250721_advanced_version_of_gemini_with_deep_think_officiall", "title": "Advanced version of Gemini with Deep Think officially achieves gold-medal standard at the International Mathematical Olympiad" }, { "disposition": "included", "id": "candidate_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "source_id": "source_candidate_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "title": "Gemini 2.5 Flash-Lite is now ready for scaled production use" }, { "disposition": "included", "id": "candidate_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250801_try_deep_think_in_the_gemini_app" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "source_id": "source_candidate_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "title": "Try Deep Think in the Gemini app" }, { "disposition": "included", "id": "candidate_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "source_id": "source_candidate_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "title": "Introducing Gemma 3 270M: The compact model for hyper-efficient AI" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250826_image_editing_in_gemini_just_got_a_major_upgrade", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250826_image_editing_in_gemini_just_got_a_major_upgrade" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-26", "reason_code": "excluded_image_or_video_only", "source_id": "source_candidate_google_deepmind_20250826_image_editing_in_gemini_just_got_a_major_upgrade", "title": "Image editing in Gemini just got a major upgrade" }, { "disposition": "included", "id": "candidate_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "source_id": "source_candidate_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "title": "VaultGemma: The world's most capable differentially private LLM" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250917_gemini_achieves_gold_medal_level_at_the_internationa", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250917_gemini_achieves_gold_medal_level_at_the_internationa" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-17", "reason_code": "not_release_announcement", "source_id": "source_candidate_google_deepmind_20250917_gemini_achieves_gold_medal_level_at_the_internationa", "title": "Gemini achieves gold-medal level at the International Collegiate Programming Contest World Finals" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20250925_gemini_robotics_1_5_brings_ai_agents_into_the_physic", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20250925_gemini_robotics_1_5_brings_ai_agents_into_the_physic" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-25", "reason_code": "excluded_robotics", "source_id": "source_candidate_google_deepmind_20250925_gemini_robotics_1_5_brings_ai_agents_into_the_physic", "title": "Gemini Robotics 1.5 brings AI agents into the physical world" }, { "disposition": "included", "id": "candidate_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "source_id": "source_candidate_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "title": "Introducing the Gemini 2.5 Computer Use model" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20251015_how_a_gemma_model_helped_discover_a_new_potential_ca", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20251015_how_a_gemma_model_helped_discover_a_new_potential_ca" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "reason_code": "not_release_announcement", "source_id": "source_candidate_google_deepmind_20251015_how_a_gemma_model_helped_discover_a_new_potential_ca", "title": "How a Gemma model helped discover a new potential cancer therapy pathway" }, { "disposition": "included", "id": "candidate_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "source_id": "source_candidate_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "title": "A new era of intelligence with Gemini 3" }, { "canonical_candidate_id": "candidate_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "disposition": "duplicate_or_alias", "id": "candidate_google_deepmind_20251118_start_building_with_gemini_3", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20251118_start_building_with_gemini_3" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "source_id": "source_candidate_google_deepmind_20251118_start_building_with_gemini_3", "title": "Start building with Gemini 3" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20251120_build_with_nano_banana_pro_our_gemini_3_pro_image_mo", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20251120_build_with_nano_banana_pro_our_gemini_3_pro_image_mo" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-20", "reason_code": "excluded_image_or_video_only", "source_id": "source_candidate_google_deepmind_20251120_build_with_nano_banana_pro_our_gemini_3_pro_image_mo", "title": "Build with Nano Banana Pro, our Gemini 3 Pro Image model" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20251120_how_we_re_bringing_ai_image_verification_to_the_gemi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20251120_how_we_re_bringing_ai_image_verification_to_the_gemi" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-20", "reason_code": "not_release_announcement", "source_id": "source_candidate_google_deepmind_20251120_how_we_re_bringing_ai_image_verification_to_the_gemi", "title": "How we’re bringing AI image verification to the Gemini app" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20251212_improved_gemini_audio_models_for_powerful_voice_expe", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20251212_improved_gemini_audio_models_for_powerful_voice_expe" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-12", "reason_code": "excluded_audio_only", "source_id": "source_candidate_google_deepmind_20251212_improved_gemini_audio_models_for_powerful_voice_expe", "title": "Improved Gemini audio models for powerful voice experiences" }, { "disposition": "included", "id": "candidate_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "source_id": "source_candidate_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "title": "Gemini 3 Flash: frontier intelligence built for speed" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20251219_gemma_scope_2_helping_the_ai_safety_community_deepen", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20251219_gemma_scope_2_helping_the_ai_safety_community_deepen" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-19", "reason_code": "not_release_announcement", "source_id": "source_candidate_google_deepmind_20251219_gemma_scope_2_helping_the_ai_safety_community_deepen", "title": "Gemma Scope 2: helping the AI safety community deepen understanding of complex language model behavior" }, { "disposition": "included", "id": "candidate_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-11", "source_id": "source_candidate_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "title": "Accelerating Mathematical and Scientific Discovery with Gemini Deep Think" }, { "disposition": "included", "id": "candidate_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "source_id": "source_candidate_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "title": "Gemini 3 Deep Think: Advancing science, research and engineering" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260218_a_new_way_to_express_yourself_gemini_can_now_create", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260218_a_new_way_to_express_yourself_gemini_can_now_create" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-18", "reason_code": "excluded_audio_only", "source_id": "source_candidate_google_deepmind_20260218_a_new_way_to_express_yourself_gemini_can_now_create", "title": "A new way to express yourself: Gemini can now create music" }, { "disposition": "included", "id": "candidate_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "source_id": "source_candidate_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "title": "Gemini 3.1 Pro: A smarter model for your most complex tasks" }, { "disposition": "included", "id": "candidate_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "source_id": "source_candidate_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "title": "Gemini 3.1 Flash-Lite: Built for intelligence at scale" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260326_gemini_3_1_flash_live_making_audio_ai_more_natural_a", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260326_gemini_3_1_flash_live_making_audio_ai_more_natural_a" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-26", "reason_code": "excluded_audio_only", "source_id": "source_candidate_google_deepmind_20260326_gemini_3_1_flash_live_making_audio_ai_more_natural_a", "title": "Gemini 3.1 Flash Live: Making audio AI more natural and reliable" }, { "disposition": "included", "id": "candidate_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "source_id": "source_candidate_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "title": "Gemma 4: Byte for byte, the most capable open models" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260414_gemini_robotics_er_1_6_powering_real_world_robotics", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260414_gemini_robotics_er_1_6_powering_real_world_robotics" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-14", "reason_code": "excluded_robotics", "source_id": "source_candidate_google_deepmind_20260414_gemini_robotics_er_1_6_powering_real_world_robotics", "title": "Gemini Robotics-ER 1.6: Powering real-world robotics tasks through enhanced embodied reasoning" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260415_gemini_3_1_flash_tts_the_next_generation_of_expressi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260415_gemini_3_1_flash_tts_the_next_generation_of_expressi" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "reason_code": "excluded_audio_only", "source_id": "source_candidate_google_deepmind_20260415_gemini_3_1_flash_tts_the_next_generation_of_expressi", "title": "Gemini 3.1 Flash TTS: the next generation of expressive AI speech" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260507_alphaevolve_how_our_gemini_powered_coding_agent_is_s", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260507_alphaevolve_how_our_gemini_powered_coding_agent_is_s" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-07", "reason_code": "not_release_announcement", "source_id": "source_candidate_google_deepmind_20260507_alphaevolve_how_our_gemini_powered_coding_agent_is_s", "title": "AlphaEvolve: How our Gemini-powered coding agent is scaling impact across fields" }, { "disposition": "included", "id": "candidate_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "source_id": "source_candidate_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "title": "Gemini 3.5: frontier intelligence with action" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260519_gemini_for_science_ai_experiments_and_tools_for_a_ne", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260519_gemini_for_science_ai_experiments_and_tools_for_a_ne" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "reason_code": "not_release_announcement", "source_id": "source_candidate_google_deepmind_20260519_gemini_for_science_ai_experiments_and_tools_for_a_ne", "title": "Gemini for Science: AI experiments and tools for a new era of discovery" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260519_introducing_gemini_omni", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260519_introducing_gemini_omni" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "reason_code": "excluded_image_or_video_only", "source_id": "source_candidate_google_deepmind_20260519_introducing_gemini_omni", "title": "Introducing Gemini Omni" }, { "disposition": "included", "id": "candidate_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "source_id": "source_candidate_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "title": "Introducing Gemma 4 12B: a unified, encoder-free multimodal model" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260609_fluid_natural_voice_translation_with_gemini_3_5_live", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260609_fluid_natural_voice_translation_with_gemini_3_5_live" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "reason_code": "excluded_audio_only", "source_id": "source_candidate_google_deepmind_20260609_fluid_natural_voice_translation_with_gemini_3_5_live", "title": "Fluid, natural voice translation with Gemini 3.5 Live Translate" }, { "disposition": "included", "id": "candidate_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-10", "source_id": "source_candidate_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "title": "DiffusionGemma: [numeric speed claim redacted] faster text generation" }, { "disposition": "included", "id": "candidate_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-24", "source_id": "source_candidate_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "title": "Introducing computer use in Gemini 3.5 Flash" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260630_start_building_with_nano_banana_2_lite_and_gemini_om", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260630_start_building_with_nano_banana_2_lite_and_gemini_om" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "reason_code": "excluded_image_or_video_only", "source_id": "source_candidate_google_deepmind_20260630_start_building_with_nano_banana_2_lite_and_gemini_om", "title": "Start building with Nano Banana 2 Lite and Gemini Omni Flash" }, { "disposition": "included", "id": "candidate_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "source_id": "source_candidate_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "title": "Introducing Gemini 3.5 Flash Cyber" }, { "disposition": "included", "id": "candidate_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "source_id": "source_candidate_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "title": "Introducing Gemini 3.6 Flash, 3.5 Flash-Lite, and 3.5 Flash Cyber" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260730_gemini_robotics_2_brings_whole_body_intelligence_to", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260730_gemini_robotics_2_brings_whole_body_intelligence_to" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "reason_code": "excluded_robotics", "source_id": "source_candidate_google_deepmind_20260730_gemini_robotics_2_brings_whole_body_intelligence_to", "title": "Gemini Robotics 2 brings whole body intelligence to robots" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260730_gemini_robotics_er_2_powering_robotics_with_video_un", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260730_gemini_robotics_er_2_powering_robotics_with_video_un" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "reason_code": "excluded_robotics", "source_id": "source_candidate_google_deepmind_20260730_gemini_robotics_er_2_powering_robotics_with_video_un", "title": "Gemini Robotics ER 2: powering robotics with video understanding, task orchestration, and multi-robot collaboration" }, { "disposition": "included", "id": "candidate_google_deepmind_20260813_introducing_gemini_3_7_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "source_id": "source_candidate_google_deepmind_20260813_introducing_gemini_3_7_flash", "title": "Introducing Gemini 3.7 Flash" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260826_intelligent_transcription_with_gemini_3_5_transcribe", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260826_intelligent_transcription_with_gemini_3_5_transcribe" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "reason_code": "excluded_audio_only", "source_id": "source_candidate_google_deepmind_20260826_intelligent_transcription_with_gemini_3_5_transcribe", "title": "Intelligent transcription with Gemini 3.5 Transcribe" }, { "disposition": "excluded", "id": "candidate_google_deepmind_20260827_gemini_omni_1_1_flash_lets_you_build_with_more_contr", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "candidate_google_deepmind_20260827_gemini_omni_1_1_flash_lets_you_build_with_more_contr" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-27", "reason_code": "excluded_image_or_video_only", "source_id": "source_candidate_google_deepmind_20260827_gemini_omni_1_1_flash_lets_you_build_with_more_contr", "title": "Gemini Omni 1.1 Flash lets you build with more control" }, { "disposition": "excluded", "id": "candidate_meta_20240215_v_jepa", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20240215_v_jepa" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "reason_code": "excluded_video_only", "source_id": "source_candidate_meta_20240215_v_jepa", "title": "V-JEPA: The next step toward advanced machine intelligence" }, { "disposition": "excluded", "id": "candidate_meta_20240320_scenescript", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20240320_scenescript" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-20", "reason_code": "excluded_image_or_3d_only", "source_id": "source_candidate_meta_20240320_scenescript", "title": "Introducing SceneScript, a novel approach for 3D scene reconstruction" }, { "disposition": "included", "id": "candidate_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "source_id": "source_candidate_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "title": "Introducing Meta Llama 3: The most capable openly available LLM to date" }, { "canonical_candidate_id": "candidate_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "disposition": "duplicate_or_alias", "id": "candidate_meta_20240418_our_responsible_approach_to_meta_ai_and_meta_llama_3", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20240418_our_responsible_approach_to_meta_ai_and_meta_llama_3" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "source_id": "source_candidate_meta_20240418_our_responsible_approach_to_meta_ai_and_meta_llama_3", "title": "Our responsible approach to Meta AI and Meta Llama 3" }, { "disposition": "included", "id": "candidate_meta_20240618_meta_fair_research_new_releases", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20240618_meta_fair_research_new_releases" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "source_id": "source_candidate_meta_20240618_meta_fair_research_new_releases", "title": "Sharing new research, models, and datasets from Meta FAIR" }, { "canonical_candidate_id": "candidate_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "disposition": "duplicate_or_alias", "id": "candidate_meta_20240723_expanding_our_open_source_large_language_models_resp", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20240723_expanding_our_open_source_large_language_models_resp" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "source_id": "source_candidate_meta_20240723_expanding_our_open_source_large_language_models_resp", "title": "Expanding our open source large language models responsibly" }, { "disposition": "included", "id": "candidate_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "source_id": "source_candidate_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "title": "Introducing Llama 3.1: Our most capable models to date" }, { "disposition": "included", "id": "candidate_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "source_id": "source_candidate_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "title": "Llama 3.2: Revolutionizing edge AI and vision with open, customizable models" }, { "disposition": "excluded", "id": "candidate_meta_20241004_meta_movie_gen", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20241004_meta_movie_gen" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-04", "reason_code": "excluded_image_video_audio_only", "source_id": "source_candidate_meta_20241004_meta_movie_gen", "title": "How Meta Movie Gen could usher in a new AI-enabled era for content creators" }, { "disposition": "included", "id": "candidate_meta_20241018_meta_fair_research_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20241018_meta_fair_research_bundle" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "source_id": "source_candidate_meta_20241018_meta_fair_research_bundle", "title": "Sharing new research, models, and datasets from Meta FAIR" }, { "disposition": "included", "id": "candidate_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "source_id": "source_candidate_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "title": "Introducing quantized Llama models with increased speed and a reduced memory footprint" }, { "disposition": "excluded", "id": "candidate_meta_20241031_embodied_ai_robotics_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20241031_embodied_ai_robotics_bundle" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-31", "reason_code": "excluded_robotics", "source_id": "source_candidate_meta_20241031_embodied_ai_robotics_bundle", "title": "Advancing embodied AI through progress in touch perception, dexterity, and human-robot interaction" }, { "disposition": "included", "id": "candidate_meta_20241212_meta_fair_agents_architecture_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20241212_meta_fair_agents_architecture_bundle" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "source_id": "source_candidate_meta_20241212_meta_fair_agents_architecture_bundle", "title": "Sharing new research, models, and datasets from Meta FAIR" }, { "disposition": "included", "id": "candidate_meta_20241219_the_future_of_ai_built_with_llama", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "source_id": "source_candidate_meta_20241219_the_future_of_ai_built_with_llama", "title": "The future of AI: Built with Llama" }, { "disposition": "excluded", "id": "candidate_meta_20250207_human_centered_research_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20250207_human_centered_research_bundle" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-07", "reason_code": "excluded_robotics_audio_specialized", "source_id": "source_candidate_meta_20250207_human_centered_research_bundle", "title": "Advancing machine intelligence through human-centered research" }, { "disposition": "excluded", "id": "candidate_meta_20250212_sam_2_1_sagemaker_update", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20250212_sam_2_1_sagemaker_update" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-12", "reason_code": "excluded_image_video_only", "source_id": "source_candidate_meta_20250212_sam_2_1_sagemaker_update", "title": "Update: Expanding access to Meta Segment Anything 2.1 on Amazon SageMaker JumpStart" }, { "disposition": "included", "id": "candidate_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "source_id": "source_candidate_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "title": "The Llama 4 herd: The beginning of a new era of natively multimodal AI innovation" }, { "disposition": "included", "id": "candidate_meta_20250417_perception_localization_reasoning_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20250417_perception_localization_reasoning_bundle" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "source_id": "source_candidate_meta_20250417_perception_localization_reasoning_bundle", "title": "Advancing AI systems through progress in perception, localization, and reasoning" }, { "disposition": "included", "id": "candidate_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "source_id": "source_candidate_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "title": "Everything we announced at our first-ever LlamaCon" }, { "disposition": "excluded", "id": "candidate_meta_20250429_llama_protection_tools", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20250429_llama_protection_tools" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "reason_code": "excluded_specialized_safety_tools", "source_id": "source_candidate_meta_20250429_llama_protection_tools", "title": "Sharing new open source protection tools and advancements in AI privacy and security" }, { "disposition": "excluded", "id": "candidate_meta_20250514_fair_science_release_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20250514_fair_science_release_bundle" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-14", "reason_code": "excluded_outside_included_model_classes", "source_id": "source_candidate_meta_20250514_fair_science_release_bundle", "title": "Sharing new breakthroughs and artifacts supporting molecular property prediction, language processing, and neuroscience" }, { "disposition": "excluded", "id": "candidate_meta_20250611_v_jepa_2", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20250611_v_jepa_2" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-11", "reason_code": "excluded_video_and_robotics", "source_id": "source_candidate_meta_20250611_v_jepa_2", "title": "Introducing the V-JEPA 2 world model and new benchmarks for physical reasoning" }, { "disposition": "excluded", "id": "candidate_meta_20250627_seamless_interaction", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20250627_seamless_interaction" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-27", "reason_code": "excluded_audio_visual_avatar_motion", "source_id": "source_candidate_meta_20250627_seamless_interaction", "title": "Modeling natural conversational dynamics with Seamless Interaction" }, { "disposition": "excluded", "id": "candidate_meta_20250814_dinov3", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20250814_dinov3" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "reason_code": "excluded_image_only", "source_id": "source_candidate_meta_20250814_dinov3", "title": "DINOv3: Self-supervised learning for vision at unprecedented scale" }, { "disposition": "excluded", "id": "candidate_meta_20251110_omnilingual_asr", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20251110_omnilingual_asr" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-10", "reason_code": "excluded_audio_only", "source_id": "source_candidate_meta_20251110_omnilingual_asr", "title": "Omnilingual ASR: Advancing Automatic Speech Recognition for [quantity redacted] Languages" }, { "disposition": "excluded", "id": "candidate_meta_20251119_sam_3d", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20251119_sam_3d" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "reason_code": "excluded_image_or_3d_only", "source_id": "source_candidate_meta_20251119_sam_3d", "title": "Introducing SAM 3D: Powerful 3D Reconstruction for Physical World Images" }, { "disposition": "excluded", "id": "candidate_meta_20251216_sam_audio", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20251216_sam_audio" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-16", "reason_code": "excluded_audio_only", "source_id": "source_candidate_meta_20251216_sam_audio", "title": "Introducing SAM Audio: The First Unified Multimodal Model for Audio Separation" }, { "disposition": "excluded", "id": "candidate_meta_20260327_sam_3_1", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20260327_sam_3_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-27", "reason_code": "excluded_image_video_only", "source_id": "source_candidate_meta_20260327_sam_3_1", "title": "SAM 3.1: Faster and More Accessible Real-Time Video Detection and Tracking With Multiplexing and Global Reasoning" }, { "disposition": "included", "id": "candidate_meta_20260408_introducing_muse_spark", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20260408_introducing_muse_spark" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "source_id": "source_candidate_meta_20260408_introducing_muse_spark", "title": "Introducing Muse Spark: Scaling Towards Personal Superintelligence" }, { "disposition": "excluded", "id": "candidate_meta_20260629_brain2qwerty", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20260629_brain2qwerty" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-29", "reason_code": "excluded_specialized_neurointerface", "source_id": "source_candidate_meta_20260629_brain2qwerty", "title": "From Brain Waves to Words: Brain2Qwerty Offers a New Path to Communication Without Surgery" }, { "disposition": "excluded", "id": "candidate_meta_20260707_introducing_muse_image_and_muse_video", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20260707_introducing_muse_image_and_muse_video" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-07", "reason_code": "excluded_image_or_video_only", "source_id": "source_candidate_meta_20260707_introducing_muse_image_and_muse_video", "title": "Introducing Muse Image and Muse Video" }, { "disposition": "included", "id": "candidate_meta_20260709_introducing_muse_spark_1_1", "lab_id": "meta", "provenance": { "record_ids": [ "candidate_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "source_id": "source_candidate_meta_20260709_introducing_muse_spark_1_1", "title": "Introducing Muse Spark 1.1" }, { "disposition": "included", "id": "candidate_openai_20240125_new_embedding_models_and_api_updates", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20240125_new_embedding_models_and_api_updates" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "source_id": "source_candidate_openai_20240125_new_embedding_models_and_api_updates", "title": "New embedding models and API updates" }, { "disposition": "included", "id": "candidate_openai_20240513_hello_gpt_4o", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20240513_hello_gpt_4o" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-13", "source_id": "source_candidate_openai_20240513_hello_gpt_4o", "title": "Hello GPT-4o" }, { "canonical_candidate_id": "candidate_openai_20240513_hello_gpt_4o", "disposition": "duplicate_or_alias", "id": "candidate_openai_20240513_introducing_gpt_4o_and_more_tools_to_chatgpt_free_users", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20240513_introducing_gpt_4o_and_more_tools_to_chatgpt_free_users" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-13", "source_id": "source_candidate_openai_20240513_introducing_gpt_4o_and_more_tools_to_chatgpt_free_users", "title": "Introducing GPT-4o and more tools to ChatGPT free users" }, { "disposition": "included", "id": "candidate_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "source_id": "source_candidate_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "title": "GPT-4o mini: advancing cost-efficient intelligence" }, { "disposition": "included", "id": "candidate_openai_20240912_introducing_openai_o1_preview", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20240912_introducing_openai_o1_preview" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "source_id": "source_candidate_openai_20240912_introducing_openai_o1_preview", "title": "Introducing OpenAI o1" }, { "disposition": "included", "id": "candidate_openai_20240912_openai_o1_mini", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20240912_openai_o1_mini" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "source_id": "source_candidate_openai_20240912_openai_o1_mini", "title": "OpenAI o1-mini" }, { "disposition": "excluded", "id": "candidate_openai_20240926_upgrading_the_moderation_api_with_our_new_multimodal_moderation_model", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20240926_upgrading_the_moderation_api_with_our_new_multimodal_moderation_model" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-26", "reason_code": "excluded_specialized_safety_model", "source_id": "source_candidate_openai_20240926_upgrading_the_moderation_api_with_our_new_multimodal_moderation_model", "title": "Upgrading the Moderation API with our new multimodal moderation model" }, { "disposition": "excluded", "id": "candidate_openai_20241001_introducing_the_realtime_api", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20241001_introducing_the_realtime_api" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-01", "reason_code": "excluded_audio_only", "source_id": "source_candidate_openai_20241001_introducing_the_realtime_api", "title": "Introducing the Realtime API" }, { "disposition": "excluded", "id": "candidate_openai_20241209_sora_is_here", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20241209_sora_is_here" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-09", "reason_code": "excluded_video_only", "source_id": "source_candidate_openai_20241209_sora_is_here", "title": "Sora is here" }, { "disposition": "included", "id": "candidate_openai_20241217_openai_o1_and_new_tools_for_developers", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "source_id": "source_candidate_openai_20241217_openai_o1_and_new_tools_for_developers", "title": "OpenAI o1 and new tools for developers" }, { "disposition": "included", "id": "candidate_openai_20250123_computer_using_agent", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250123_computer_using_agent" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "source_id": "source_candidate_openai_20250123_computer_using_agent", "title": "Computer-Using Agent" }, { "canonical_candidate_id": "candidate_openai_20250123_computer_using_agent", "disposition": "duplicate_or_alias", "id": "candidate_openai_20250123_introducing_operator", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250123_introducing_operator" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "source_id": "source_candidate_openai_20250123_introducing_operator", "title": "Introducing Operator" }, { "disposition": "included", "id": "candidate_openai_20250131_openai_o3_mini", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250131_openai_o3_mini" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "source_id": "source_candidate_openai_20250131_openai_o3_mini", "title": "OpenAI o3-mini" }, { "disposition": "included", "id": "candidate_openai_20250227_introducing_gpt_4_5", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250227_introducing_gpt_4_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "source_id": "source_candidate_openai_20250227_introducing_gpt_4_5", "title": "Introducing GPT-4.5" }, { "disposition": "excluded", "id": "candidate_openai_20250320_introducing_next_generation_audio_models_in_the_api", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250320_introducing_next_generation_audio_models_in_the_api" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-20", "reason_code": "excluded_audio_only", "source_id": "source_candidate_openai_20250320_introducing_next_generation_audio_models_in_the_api", "title": "Introducing next-generation audio models in the API" }, { "disposition": "excluded", "id": "candidate_openai_20250325_introducing_4o_image_generation", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250325_introducing_4o_image_generation" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "reason_code": "excluded_image_only", "source_id": "source_candidate_openai_20250325_introducing_4o_image_generation", "title": "Introducing 4o Image Generation" }, { "disposition": "included", "id": "candidate_openai_20250414_introducing_gpt_4_1_in_the_api", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "source_id": "source_candidate_openai_20250414_introducing_gpt_4_1_in_the_api", "title": "Introducing GPT-4.1 in the API" }, { "disposition": "included", "id": "candidate_openai_20250416_introducing_openai_o3_and_o4_mini", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "source_id": "source_candidate_openai_20250416_introducing_openai_o3_and_o4_mini", "title": "Introducing OpenAI o3 and o4-mini" }, { "disposition": "excluded", "id": "candidate_openai_20250423_introducing_our_latest_image_generation_model_in_the_api", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250423_introducing_our_latest_image_generation_model_in_the_api" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-23", "reason_code": "excluded_image_only", "source_id": "source_candidate_openai_20250423_introducing_our_latest_image_generation_model_in_the_api", "title": "Introducing our latest image generation model in the API" }, { "disposition": "included", "id": "candidate_openai_20250516_introducing_codex", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250516_introducing_codex" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-16", "source_id": "source_candidate_openai_20250516_introducing_codex", "title": "Introducing Codex" }, { "disposition": "included", "id": "candidate_openai_20250805_introducing_gpt_oss", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250805_introducing_gpt_oss" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "source_id": "source_candidate_openai_20250805_introducing_gpt_oss", "title": "Introducing gpt-oss" }, { "disposition": "included", "id": "candidate_openai_20250807_introducing_gpt_5", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250807_introducing_gpt_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "source_id": "source_candidate_openai_20250807_introducing_gpt_5", "title": "Introducing GPT-5" }, { "canonical_candidate_id": "candidate_openai_20250807_introducing_gpt_5", "disposition": "duplicate_or_alias", "id": "candidate_openai_20250807_introducing_gpt_5_for_developers", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250807_introducing_gpt_5_for_developers" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "source_id": "source_candidate_openai_20250807_introducing_gpt_5_for_developers", "title": "Introducing GPT‑5 for developers" }, { "disposition": "excluded", "id": "candidate_openai_20250828_introducing_gpt_realtime_and_realtime_api_updates_fo", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250828_introducing_gpt_realtime_and_realtime_api_updates_fo" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-28", "reason_code": "excluded_audio_only", "source_id": "source_candidate_openai_20250828_introducing_gpt_realtime_and_realtime_api_updates_fo", "title": "Introducing gpt-realtime and Realtime API updates for production voice agents" }, { "disposition": "included", "id": "candidate_openai_20250915_introducing_upgrades_to_codex", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250915_introducing_upgrades_to_codex" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "source_id": "source_candidate_openai_20250915_introducing_upgrades_to_codex", "title": "Introducing upgrades to Codex" }, { "disposition": "excluded", "id": "candidate_openai_20250930_sora_2_is_here", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20250930_sora_2_is_here" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-30", "reason_code": "excluded_video_audio_only", "source_id": "source_candidate_openai_20250930_sora_2_is_here", "title": "Sora 2 is here" }, { "disposition": "excluded", "id": "candidate_openai_20251029_introducing_gpt_oss_safeguard", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20251029_introducing_gpt_oss_safeguard" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-29", "reason_code": "excluded_specialized_safety_model", "source_id": "source_candidate_openai_20251029_introducing_gpt_oss_safeguard", "title": "Introducing gpt-oss-safeguard" }, { "disposition": "included", "id": "candidate_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "source_id": "source_candidate_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "title": "GPT-5.1: A smarter, more conversational ChatGPT" }, { "canonical_candidate_id": "candidate_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "disposition": "duplicate_or_alias", "id": "candidate_openai_20251113_introducing_gpt_5_1_for_developers", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20251113_introducing_gpt_5_1_for_developers" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-13", "source_id": "source_candidate_openai_20251113_introducing_gpt_5_1_for_developers", "title": "Introducing GPT-5.1 for developers" }, { "disposition": "included", "id": "candidate_openai_20251119_building_more_with_gpt_5_1_codex_max", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "source_id": "source_candidate_openai_20251119_building_more_with_gpt_5_1_codex_max", "title": "Building more with GPT-5.1-Codex-Max" }, { "canonical_candidate_id": "candidate_openai_20251211_introducing_gpt_5_2", "disposition": "duplicate_or_alias", "id": "candidate_openai_20251211_advancing_science_and_math_with_gpt_5_2", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20251211_advancing_science_and_math_with_gpt_5_2" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "source_id": "source_candidate_openai_20251211_advancing_science_and_math_with_gpt_5_2", "title": "Advancing science and math with GPT-5.2" }, { "disposition": "included", "id": "candidate_openai_20251211_introducing_gpt_5_2", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20251211_introducing_gpt_5_2" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "source_id": "source_candidate_openai_20251211_introducing_gpt_5_2", "title": "Introducing GPT-5.2" }, { "disposition": "excluded", "id": "candidate_openai_20251216_the_new_chatgpt_images_is_here", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20251216_the_new_chatgpt_images_is_here" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-16", "reason_code": "excluded_image_only", "source_id": "source_candidate_openai_20251216_the_new_chatgpt_images_is_here", "title": "The new ChatGPT Images is here" }, { "disposition": "included", "id": "candidate_openai_20251218_introducing_gpt_5_2_codex", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "source_id": "source_candidate_openai_20251218_introducing_gpt_5_2_codex", "title": "Introducing GPT-5.2-Codex" }, { "disposition": "included", "id": "candidate_openai_20260205_introducing_gpt_5_3_codex", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "source_id": "source_candidate_openai_20260205_introducing_gpt_5_3_codex", "title": "Introducing GPT-5.3-Codex" }, { "disposition": "included", "id": "candidate_openai_20260212_introducing_gpt_5_3_codex_spark", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260212_introducing_gpt_5_3_codex_spark" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "source_id": "source_candidate_openai_20260212_introducing_gpt_5_3_codex_spark", "title": "Introducing GPT-5.3-Codex-Spark" }, { "disposition": "included", "id": "candidate_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "source_id": "source_candidate_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "title": "GPT-5.3 Instant: Smoother, more useful everyday conversations" }, { "disposition": "included", "id": "candidate_openai_20260305_introducing_gpt_5_4", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260305_introducing_gpt_5_4" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "source_id": "source_candidate_openai_20260305_introducing_gpt_5_4", "title": "Introducing GPT-5.4" }, { "disposition": "included", "id": "candidate_openai_20260317_introducing_gpt_5_4_mini_and_nano", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "source_id": "source_candidate_openai_20260317_introducing_gpt_5_4_mini_and_nano", "title": "Introducing GPT-5.4 mini and nano" }, { "disposition": "included", "id": "candidate_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "source_id": "source_candidate_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "title": "Introducing GPT-Rosalind for life sciences research" }, { "disposition": "excluded", "id": "candidate_openai_20260421_introducing_chatgpt_images_2_0", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260421_introducing_chatgpt_images_2_0" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-21", "reason_code": "excluded_image_only", "source_id": "source_candidate_openai_20260421_introducing_chatgpt_images_2_0", "title": "Introducing ChatGPT Images 2.0" }, { "disposition": "included", "id": "candidate_openai_20260423_introducing_gpt_5_5", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260423_introducing_gpt_5_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "source_id": "source_candidate_openai_20260423_introducing_gpt_5_5", "title": "Introducing GPT-5.5" }, { "disposition": "included", "id": "candidate_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "source_id": "source_candidate_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "title": "GPT-5.5 Instant: smarter, clearer, and more personalized" }, { "disposition": "excluded", "id": "candidate_openai_20260507_advancing_voice_intelligence_with_new_models_in_the_api", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260507_advancing_voice_intelligence_with_new_models_in_the_api" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-07", "reason_code": "excluded_audio_only", "source_id": "source_candidate_openai_20260507_advancing_voice_intelligence_with_new_models_in_the_api", "title": "Advancing voice intelligence with new models in the API" }, { "disposition": "included", "id": "candidate_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "source_id": "source_candidate_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "title": "Introducing new capabilities to GPT-Rosalind" }, { "disposition": "included", "id": "candidate_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "source_id": "source_candidate_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "title": "Previewing GPT-5.6 Sol: a next-generation model" }, { "disposition": "excluded", "id": "candidate_openai_20260708_introducing_gpt_live", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260708_introducing_gpt_live" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-08", "reason_code": "excluded_audio_only", "source_id": "source_candidate_openai_20260708_introducing_gpt_live", "title": "Introducing GPT-Live" }, { "disposition": "included", "id": "candidate_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "source_id": "source_candidate_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "title": "GPT-5.6: Frontier intelligence that scales with your ambition" }, { "disposition": "included", "id": "candidate_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "source_id": "source_candidate_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "title": "Advancing the price-performance frontier with GPT-5.6" }, { "disposition": "included", "id": "candidate_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "source_id": "source_candidate_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "title": "Improving GPT‑5.6 Sol in ChatGPT—and expanding access to GPT-5.6 Luna for free users" }, { "disposition": "included", "id": "candidate_openai_20260811_daybreak_models_are_now_available_on_aws", "lab_id": "openai", "provenance": { "record_ids": [ "candidate_openai_20260811_daybreak_models_are_now_available_on_aws" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-11", "source_id": "source_candidate_openai_20260811_daybreak_models_are_now_available_on_aws", "title": "Daybreak models are now available on AWS" }, { "disposition": "included", "id": "candidate_qwen_20240123_introducing_qwen_qwen", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240123_introducing_qwen_qwen" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "source_id": "source_candidate_qwen_20240123_introducing_qwen_qwen", "title": "Introducing Qwen" }, { "disposition": "included", "id": "candidate_qwen_20240125_introducing_qwen_vl_qwen_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "source_id": "source_candidate_qwen_20240125_introducing_qwen_vl_qwen_vl", "title": "Introducing Qwen-VL" }, { "disposition": "included", "id": "candidate_qwen_20240204_introducing_qwen1_5_qwen1_5", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "source_id": "source_candidate_qwen_20240204_introducing_qwen1_5_qwen1_5", "title": "Introducing Qwen1.5" }, { "disposition": "included", "id": "candidate_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "source_id": "source_candidate_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "title": "Qwen1.5-MoE: Matching 7B Model Performance with [activation ratio redacted]" }, { "disposition": "included", "id": "candidate_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "source_id": "source_candidate_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "title": "Qwen1.5-32B: Fitting the Capstone of the Qwen1.5 Language Model Series" }, { "disposition": "included", "id": "candidate_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "source_id": "source_candidate_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "title": "Code with CodeQwen1.5" }, { "disposition": "included", "id": "candidate_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "source_id": "source_candidate_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "title": "Qwen1.5-110B: The First 100B+ Model of the Qwen1.5 Series" }, { "disposition": "included", "id": "candidate_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-11", "source_id": "source_candidate_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "title": "Notes on Qwen-Max-0428" }, { "disposition": "excluded", "id": "candidate_qwen_20240606_generalizing_an_llm_from_8k_to_1m_context_using_qwen_qwen_agent_2405", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240606_generalizing_an_llm_from_8k_to_1m_context_using_qwen_qwen_agent_2405" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "reason_code": "not_release_announcement", "source_id": "source_candidate_qwen_20240606_generalizing_an_llm_from_8k_to_1m_context_using_qwen_qwen_agent_2405", "title": "Generalizing an LLM from 8k to 1M Context using Qwen-Agent" }, { "disposition": "included", "id": "candidate_qwen_20240606_hello_qwen2_qwen2", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "source_id": "source_candidate_qwen_20240606_hello_qwen2_qwen2", "title": "Hello Qwen2" }, { "disposition": "included", "id": "candidate_qwen_20240807_introducing_qwen2_math_qwen2_math", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "source_id": "source_candidate_qwen_20240807_introducing_qwen2_math_qwen2_math", "title": "Introducing Qwen2-Math" }, { "disposition": "excluded", "id": "candidate_qwen_20240809_qwen2_audio_chat_with_your_voice_qwen2_audio", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240809_qwen2_audio_chat_with_your_voice_qwen2_audio" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-09", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20240809_qwen2_audio_chat_with_your_voice_qwen2_audio", "title": "Qwen2-Audio: Chat with Your Voice!" }, { "disposition": "included", "id": "candidate_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "source_id": "source_candidate_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "title": "Qwen2-VL: To See the World More Clearly" }, { "disposition": "included", "id": "candidate_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "source_id": "source_candidate_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "title": "Qwen2.5: A Party of Foundation Models!" }, { "disposition": "included", "id": "candidate_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "source_id": "source_candidate_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "title": "Qwen2.5-Coder: Code More, Learn More!" }, { "disposition": "included", "id": "candidate_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "source_id": "source_candidate_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "title": "Qwen2.5-LLM: Extending the boundary of LLMs" }, { "disposition": "included", "id": "candidate_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "source_id": "source_candidate_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "title": "Qwen2.5-Math: The world's leading open-sourced mathematical LLMs" }, { "disposition": "included", "id": "candidate_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "source_id": "source_candidate_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "title": "Qwen2.5-Coder Series: Powerful, Diverse, Practical." }, { "disposition": "included", "id": "candidate_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "source_id": "source_candidate_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "title": "Extending the Context Length to 1M Tokens!" }, { "disposition": "included", "id": "candidate_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "source_id": "source_candidate_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "title": "QwQ: Reflect Deeply on the Boundaries of the Unknown" }, { "disposition": "included", "id": "candidate_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "source_id": "source_candidate_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "title": "QVQ: To See the World with Wisdom" }, { "disposition": "excluded", "id": "candidate_qwen_20250113_towards_effective_process_supervision_in_mathematica_qwen2_5_math_prm", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250113_towards_effective_process_supervision_in_mathematica_qwen2_5_math_prm" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-13", "reason_code": "excluded_not_included_model_class", "source_id": "source_candidate_qwen_20250113_towards_effective_process_supervision_in_mathematica_qwen2_5_math_prm", "title": "Towards Effective Process Supervision in Mathematical Reasoning" }, { "disposition": "excluded", "id": "candidate_qwen_20250120_global_batch_load_balance_almost_free_lunch_to_impro_global_load_balance", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250120_global_batch_load_balance_almost_free_lunch_to_impro_global_load_balance" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "reason_code": "not_release_announcement", "source_id": "source_candidate_qwen_20250120_global_batch_load_balance_almost_free_lunch_to_impro_global_load_balance", "title": "Global-batch load balance almost free lunch to improve your MoE LLM training" }, { "disposition": "included", "id": "candidate_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "source_id": "source_candidate_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "title": "Qwen2.5-1M: Deploy Your Own Qwen with Context Length up to 1M Tokens" }, { "disposition": "included", "id": "candidate_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "source_id": "source_candidate_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "title": "Qwen2.5 VL! Qwen2.5 VL! Qwen2.5 VL!" }, { "disposition": "included", "id": "candidate_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "source_id": "source_candidate_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "title": "Qwen2.5-Max: Exploring the Intelligence of Large-scale MoE Model" }, { "disposition": "included", "id": "candidate_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "source_id": "source_candidate_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "title": "... QwQ-Max-Preview" }, { "disposition": "included", "id": "candidate_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "source_id": "source_candidate_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "title": "QwQ-32B: Embracing the Power of Reinforcement Learning" }, { "disposition": "included", "id": "candidate_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "source_id": "source_candidate_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "title": "Qwen2.5-VL-32B: Smarter and Lighter" }, { "disposition": "included", "id": "candidate_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "source_id": "source_candidate_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "title": "Qwen2.5 Omni: See, Hear, Talk, Write, Do It All!" }, { "disposition": "included", "id": "candidate_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-27", "source_id": "source_candidate_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "title": "QVQ-Max: Think with Evidence" }, { "canonical_candidate_id": "candidate_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "disposition": "duplicate_or_alias", "id": "candidate_qwen_20250428_qwen3_think_deeper_act_faster_dbf896e521a5dc531a9b9cdd74f487d8908569fe_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250428_qwen3_think_deeper_act_faster_dbf896e521a5dc531a9b9cdd74f487d8908569fe_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "source_id": "source_candidate_qwen_20250428_qwen3_think_deeper_act_faster_dbf896e521a5dc531a9b9cdd74f487d8908569fe_alias_1", "title": "Qwen3: Think Deeper, Act Faster" }, { "disposition": "included", "id": "candidate_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "source_id": "source_candidate_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "title": "Qwen3: Think Deeper, Act Faster" }, { "disposition": "excluded", "id": "candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_64e75147c1b7c0a4f63f54113a11594ae10c6142_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_64e75147c1b7c0a4f63f54113a11594ae10c6142_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-05", "reason_code": "excluded_embedding_only", "source_id": "source_candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_64e75147c1b7c0a4f63f54113a11594ae10c6142_alias_1", "title": "Qwen3 Embedding: Advancing Text Embedding and Reranking Through Foundation Models" }, { "disposition": "excluded", "id": "candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_qwen3_embedding", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_qwen3_embedding" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-05", "reason_code": "excluded_embedding_only", "source_id": "source_candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_qwen3_embedding", "title": "Qwen3 Embedding: Advancing Text Embedding and Reranking Through Foundation Models" }, { "canonical_candidate_id": "candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "disposition": "duplicate_or_alias", "id": "candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_bd66219782efce3b6337f14c61f546c89e13d37b_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_bd66219782efce3b6337f14c61f546c89e13d37b_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "source_id": "source_candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_bd66219782efce3b6337f14c61f546c89e13d37b_alias_1", "title": "Qwen VLo: From \"Understanding\" the World to \"Depicting\" It" }, { "disposition": "included", "id": "candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "source_id": "source_candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "title": "Qwen VLo: From \"Understanding\" the World to \"Depicting\" It" }, { "disposition": "excluded", "id": "candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_aeb9b63d20561045bb9359cb3fa786f397708f29_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_aeb9b63d20561045bb9359cb3fa786f397708f29_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-27", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_aeb9b63d20561045bb9359cb3fa786f397708f29_alias_1", "title": "Time to Speak Some Dialects, Qwen-TTS!" }, { "disposition": "excluded", "id": "candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_qwen_tts", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_qwen_tts" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-27", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_qwen_tts", "title": "Time to Speak Some Dialects, Qwen-TTS!" }, { "canonical_candidate_id": "candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "disposition": "duplicate_or_alias", "id": "candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_a654383695200a86b635aae18f9ae6ccba7b20da_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_a654383695200a86b635aae18f9ae6ccba7b20da_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "source_id": "source_candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_a654383695200a86b635aae18f9ae6ccba7b20da_alias_1", "title": "Qwen3-Coder: Agentic Coding in the World" }, { "disposition": "included", "id": "candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "source_id": "source_candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "title": "Qwen3-Coder: Agentic Coding in the World" }, { "disposition": "excluded", "id": "candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_d3913d3fad8cd06ee020225844573b31de982e19_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_d3913d3fad8cd06ee020225844573b31de982e19_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-24", "reason_code": "excluded_specialized_translation_model", "source_id": "source_candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_d3913d3fad8cd06ee020225844573b31de982e19_alias_1", "title": "Qwen-MT: Where Speed Meets Smart Translation" }, { "disposition": "excluded", "id": "candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_qwen_mt", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_qwen_mt" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-24", "reason_code": "excluded_specialized_translation_model", "source_id": "source_candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_qwen_mt", "title": "Qwen-MT: Where Speed Meets Smart Translation" }, { "disposition": "excluded", "id": "candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_b42c93786a6c8dbf729ec3a41636fda88b04d6e9_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_b42c93786a6c8dbf729ec3a41636fda88b04d6e9_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-27", "reason_code": "not_release_announcement", "source_id": "source_candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_b42c93786a6c8dbf729ec3a41636fda88b04d6e9_alias_1", "title": "GSPO: Towards Scalable Reinforcement Learning for Language Models" }, { "disposition": "excluded", "id": "candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_gspo", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_gspo" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-27", "reason_code": "not_release_announcement", "source_id": "source_candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_gspo", "title": "GSPO: Towards Scalable Reinforcement Learning for Language Models" }, { "disposition": "excluded", "id": "candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_51eec55b76cc7760326919ad5ab4987823ca3a4f_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_51eec55b76cc7760326919ad5ab4987823ca3a4f_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-04", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_51eec55b76cc7760326919ad5ab4987823ca3a4f_alias_1", "title": "Qwen-Image: Crafting with Native Text Rendering" }, { "disposition": "excluded", "id": "candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_qwen_image", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_qwen_image" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-04", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_qwen_image", "title": "Qwen-Image: Crafting with Native Text Rendering" }, { "disposition": "excluded", "id": "candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_3658ef85328511103029b904cb0f3fda78187735_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_3658ef85328511103029b904cb0f3fda78187735_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-18", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_3658ef85328511103029b904cb0f3fda78187735_alias_1", "title": "Qwen-Image-Edit: Image Editing with Higher Quality and Efficiency" }, { "disposition": "excluded", "id": "candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_qwen_image_edit", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_qwen_image_edit" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-18", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_qwen_image_edit", "title": "Qwen-Image-Edit: Image Editing with Higher Quality and Efficiency" }, { "disposition": "excluded", "id": "candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_41e4c0f6175f9b004a03a07e42343eaaf48329e7_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_41e4c0f6175f9b004a03a07e42343eaaf48329e7_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-08", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_41e4c0f6175f9b004a03a07e42343eaaf48329e7_alias_1", "title": "Qwen3 ASR: Hear clearly, transcribe smartly." }, { "disposition": "excluded", "id": "candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_qwen3_asr_flash", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_qwen3_asr_flash" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-08", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_qwen3_asr_flash", "title": "Qwen3 ASR: Hear clearly, transcribe smartly." }, { "canonical_candidate_id": "candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "disposition": "duplicate_or_alias", "id": "candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_4074cca80393150c248e508aa62983f9cb7d27cd_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_4074cca80393150c248e508aa62983f9cb7d27cd_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "source_id": "source_candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_4074cca80393150c248e508aa62983f9cb7d27cd_alias_1", "title": "Qwen3-Next: Towards Ultimate Training & Inference Efficiency" }, { "disposition": "included", "id": "candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "source_id": "source_candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "title": "Qwen3-Next: Towards Ultimate Training & Inference Efficiency" }, { "canonical_candidate_id": "candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "disposition": "duplicate_or_alias", "id": "candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_65f766fc2dcba7905c1cb69cc4cab90e94126bf4_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_65f766fc2dcba7905c1cb69cc4cab90e94126bf4_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "source_id": "source_candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_65f766fc2dcba7905c1cb69cc4cab90e94126bf4_alias_1", "title": "Qwen3-Omni: Natively Omni-Modal Foundation Models!" }, { "disposition": "included", "id": "candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "source_id": "source_candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "title": "Qwen3-Omni: Natively Omni-Modal Foundation Models!" }, { "disposition": "excluded", "id": "candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_b4264e11fb80b5e37350790121baf0a0f10daf82_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_b4264e11fb80b5e37350790121baf0a0f10daf82_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_b4264e11fb80b5e37350790121baf0a0f10daf82_alias_1", "title": "Qwen3-TTS-Flash: Multi-timbre & Multi-lingual & Multi-dialect Speech Synthesis." }, { "disposition": "excluded", "id": "candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_qwen3_tts", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_qwen3_tts" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_qwen3_tts", "title": "Qwen3-TTS-Flash: Multi-timbre & Multi-lingual & Multi-dialect Speech Synthesis." }, { "disposition": "excluded", "id": "candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_b2de6ae8555599bf3b87eec55a285cdf496b78e4_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_b2de6ae8555599bf3b87eec55a285cdf496b78e4_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "reason_code": "excluded_specialized_translation_model", "source_id": "source_candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_b2de6ae8555599bf3b87eec55a285cdf496b78e4_alias_1", "title": "Qwen3‑LiveTranslate: Real‑Time Multimodal Interpretation — See It, Hear It, Speak It！" }, { "disposition": "excluded", "id": "candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_qwen3_livetranslate", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_qwen3_livetranslate" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "reason_code": "excluded_specialized_translation_model", "source_id": "source_candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_qwen3_livetranslate", "title": "Qwen3‑LiveTranslate: Real‑Time Multimodal Interpretation — See It, Hear It, Speak It！" }, { "canonical_candidate_id": "candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "disposition": "duplicate_or_alias", "id": "candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_99f0335c4ad9ff6153e517418d48535ab6d8afef_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_99f0335c4ad9ff6153e517418d48535ab6d8afef_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "source_id": "source_candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_99f0335c4ad9ff6153e517418d48535ab6d8afef_alias_1", "title": "Qwen3-VL: Sharper Vision, Deeper Thought, Broader Action" }, { "disposition": "included", "id": "candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "source_id": "source_candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "title": "Qwen3-VL: Sharper Vision, Deeper Thought, Broader Action" }, { "disposition": "excluded", "id": "candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_3e4179176f4f2b028b342fb0193c192046ca291b_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_3e4179176f4f2b028b342fb0193c192046ca291b_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "reason_code": "excluded_specialized_safety_model", "source_id": "source_candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_3e4179176f4f2b028b342fb0193c192046ca291b_alias_1", "title": "Qwen3Guard: Real-time Safety for Your Token Stream" }, { "disposition": "excluded", "id": "candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_qwen3guard", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_qwen3guard" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "reason_code": "excluded_specialized_safety_model", "source_id": "source_candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_qwen3guard", "title": "Qwen3Guard: Real-time Safety for Your Token Stream" }, { "disposition": "excluded", "id": "candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_7a90090115ee193ce6a7f619522771dd9696dd93_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_7a90090115ee193ce6a7f619522771dd9696dd93_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_7a90090115ee193ce6a7f619522771dd9696dd93_alias_1", "title": "Qwen-Image-Edit-2509: Multi-Image Support, Improved Consistency" }, { "disposition": "excluded", "id": "candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_qwen_image_edit_2509", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_qwen_image_edit_2509" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_qwen_image_edit_2509", "title": "Qwen-Image-Edit-2509: Multi-Image Support, Improved Consistency" }, { "disposition": "excluded", "id": "candidate_qwen_20250922_travel_planner_your_smart_travel_designer_6228c39fa97ce1a9e65f9e7ce98787a743007fe4_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250922_travel_planner_your_smart_travel_designer_6228c39fa97ce1a9e65f9e7ce98787a743007fe4_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "reason_code": "not_release_announcement", "source_id": "source_candidate_qwen_20250922_travel_planner_your_smart_travel_designer_6228c39fa97ce1a9e65f9e7ce98787a743007fe4_alias_1", "title": "Travel Planner: Your Smart Travel Designer" }, { "disposition": "excluded", "id": "candidate_qwen_20250922_travel_planner_your_smart_travel_designer_agent", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250922_travel_planner_your_smart_travel_designer_agent" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "reason_code": "not_release_announcement", "source_id": "source_candidate_qwen_20250922_travel_planner_your_smart_travel_designer_agent", "title": "Travel Planner: Your Smart Travel Designer" }, { "canonical_candidate_id": "candidate_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "disposition": "duplicate_or_alias", "id": "candidate_qwen_20250924_qwen3_max_just_scale_it_241398b9cd6353de490b0f82806c7848c5d2777d_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250924_qwen3_max_just_scale_it_241398b9cd6353de490b0f82806c7848c5d2777d_alias_1" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "source_id": "source_candidate_qwen_20250924_qwen3_max_just_scale_it_241398b9cd6353de490b0f82806c7848c5d2777d_alias_1", "title": "Qwen3-Max: Just Scale it" }, { "disposition": "included", "id": "candidate_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "source_id": "source_candidate_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "title": "Qwen3-Max: Just Scale it" }, { "disposition": "excluded", "id": "candidate_qwen_20251112_qwen_deepresearch_when_inspiration_becomes_its_own_r_qwen_deepresearch", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20251112_qwen_deepresearch_when_inspiration_becomes_its_own_r_qwen_deepresearch" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "reason_code": "not_release_announcement", "source_id": "source_candidate_qwen_20251112_qwen_deepresearch_when_inspiration_becomes_its_own_r_qwen_deepresearch", "title": "Qwen DeepResearch: When Inspiration Becomes Its Own Reason" }, { "disposition": "excluded", "id": "candidate_qwen_20251204_qwen3_tts_update_49_timbres_10_languages_9_dialects_qwen3_tts_1128", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20251204_qwen3_tts_update_49_timbres_10_languages_9_dialects_qwen3_tts_1128" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-04", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20251204_qwen3_tts_update_49_timbres_10_languages_9_dialects_qwen3_tts_1128", "title": "Qwen3-TTS Update! 49 Timbres + 10 Languages + 9 Dialects" }, { "disposition": "excluded", "id": "candidate_qwen_20251204_sapo_a_stable_and_performant_reinforcement_learning_sapo", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20251204_sapo_a_stable_and_performant_reinforcement_learning_sapo" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-04", "reason_code": "not_release_announcement", "source_id": "source_candidate_qwen_20251204_sapo_a_stable_and_performant_reinforcement_learning_sapo", "title": "SAPO: A Stable and Performant Reinforcement Learning Method for Training Large Language Models" }, { "disposition": "included", "id": "candidate_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "source_id": "source_candidate_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "title": "Qwen3-Omni-Flash-2025-12-01：Hear You. See You. Follow Smarter!" }, { "disposition": "excluded", "id": "candidate_qwen_20251219_qwen_image_layered_layered_decomposition_for_inheren_qwen_image_layered", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20251219_qwen_image_layered_layered_decomposition_for_inheren_qwen_image_layered" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-19", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20251219_qwen_image_layered_layered_decomposition_for_inheren_qwen_image_layered", "title": "Qwen-Image-Layered: Layered Decomposition for Inherent Editablity" }, { "disposition": "excluded", "id": "candidate_qwen_20251222_qwen3_tts_steps_up_voice_cloning_and_voice_design_qwen3_tts_vc_voicedesign", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20251222_qwen3_tts_steps_up_voice_cloning_and_voice_design_qwen3_tts_vc_voicedesign" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-22", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20251222_qwen3_tts_steps_up_voice_cloning_and_voice_design_qwen3_tts_vc_voicedesign", "title": "Qwen3-TTS Steps Up: Voice Cloning and Voice Design!" }, { "disposition": "excluded", "id": "candidate_qwen_20251223_qwen_image_edit_2511_improve_consistency_qwen_image_edit_2511", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20251223_qwen_image_edit_2511_improve_consistency_qwen_image_edit_2511" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-23", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20251223_qwen_image_edit_2511_improve_consistency_qwen_image_edit_2511", "title": "Qwen-Image-Edit-2511: Improve Consistency" }, { "disposition": "excluded", "id": "candidate_qwen_20251231_qwen_image_2512", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20251231_qwen_image_2512" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-31", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20251231_qwen_image_2512", "title": "Qwen-Image-2512: Finer Details, Greater Realism" }, { "disposition": "excluded", "id": "candidate_qwen_20260108_qwen3_vl_embedding", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260108_qwen3_vl_embedding" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-08", "reason_code": "excluded_embedding_only", "source_id": "source_candidate_qwen_20260108_qwen3_vl_embedding", "title": "Qwen3-VL-Embedding and Qwen3-VL-Reranker: For the Next Generation of Multimodal Retrieval" }, { "disposition": "excluded", "id": "candidate_qwen_20260122_qwen3tts_0115", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260122_qwen3tts_0115" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-22", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20260122_qwen3tts_0115", "title": "Qwen3-TTS Family is Now Open Sourced: Voice Design, Clone, and Generation!" }, { "disposition": "included", "id": "candidate_qwen_20260126_qwen3_max_thinking", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260126_qwen3_max_thinking" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "source_id": "source_candidate_qwen_20260126_qwen3_max_thinking", "title": "Pushing Qwen3-Max-Thinking Beyond its Limits" }, { "disposition": "excluded", "id": "candidate_qwen_20260129_qwen3asr", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260129_qwen3asr" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-29", "reason_code": "excluded_audio_only", "source_id": "source_candidate_qwen_20260129_qwen3asr", "title": "Qwen3-ASR & Qwen3-ForcedAligner is Now Open Sourced: Robust, Streaming and Multilingual!" }, { "disposition": "included", "id": "candidate_qwen_20260203_qwen3_coder_next", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260203_qwen3_coder_next" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "source_id": "source_candidate_qwen_20260203_qwen3_coder_next", "title": "Qwen3-Coder-Next: Pushing Small Hybrid Models on Agentic Coding" }, { "disposition": "excluded", "id": "candidate_qwen_20260210_qwen_image_2_0", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260210_qwen_image_2_0" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-10", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20260210_qwen_image_2_0", "title": "Qwen-Image-2.0: Professional infographics, exquisite photorealism" }, { "disposition": "included", "id": "candidate_qwen_20260216_qwen3_5", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260216_qwen3_5" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "source_id": "source_candidate_qwen_20260216_qwen3_5", "title": "Qwen3.5: Towards Native Multimodal Agents" }, { "disposition": "included", "id": "candidate_qwen_20260319_qwen3_5_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260319_qwen3_5_max_preview" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-19", "source_id": "source_candidate_qwen_20260319_qwen3_5_max_preview", "title": "Qwen3.5-Max-Preview Now Available on Arena" }, { "disposition": "included", "id": "candidate_qwen_20260330_qwen3_5_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260330_qwen3_5_omni" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "source_id": "source_candidate_qwen_20260330_qwen3_5_omni", "title": "Qwen3.5-Omni: Scaling Up, Toward Native Omni-Modal AGI" }, { "disposition": "included", "id": "candidate_qwen_20260402_qwen3_6", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260402_qwen3_6" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "source_id": "source_candidate_qwen_20260402_qwen3_6", "title": "Qwen3.6-Plus: Towards Real World Agents" }, { "disposition": "included", "id": "candidate_qwen_20260415_qwen3_6_35b_a3b", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "source_id": "source_candidate_qwen_20260415_qwen3_6_35b_a3b", "title": "Qwen3.6-35B-A3B: Agentic Coding Power, Now Open to All" }, { "disposition": "included", "id": "candidate_qwen_20260418_qwen3_6_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260418_qwen3_6_max_preview" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "source_id": "source_candidate_qwen_20260418_qwen3_6_max_preview", "title": "Qwen3.6-Max-Preview: Smarter, Sharper, Still Evolving" }, { "disposition": "included", "id": "candidate_qwen_20260422_qwen3_6_27b", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260422_qwen3_6_27b" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "source_id": "source_candidate_qwen_20260422_qwen3_6_27b", "title": "Qwen3.6-27B: Flagship-Level Coding in a 27B Dense Model" }, { "disposition": "excluded", "id": "candidate_qwen_20260428_flashqla", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260428_flashqla" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-28", "reason_code": "not_release_announcement", "source_id": "source_candidate_qwen_20260428_flashqla", "title": "FlashQLA: CP-/Bwd-Friendly Fused Linear Attention Kernels for GDN" }, { "disposition": "excluded", "id": "candidate_qwen_20260430_qwen_scope", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260430_qwen_scope" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-30", "reason_code": "not_release_announcement", "source_id": "source_candidate_qwen_20260430_qwen_scope", "title": "Qwen-Scope: Decoding Intelligence, Unleashing Potential" }, { "disposition": "excluded", "id": "candidate_qwen_20260519_qwen3_5_livetranslate", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260519_qwen3_5_livetranslate" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "reason_code": "excluded_specialized_translation_model", "source_id": "source_candidate_qwen_20260519_qwen3_5_livetranslate", "title": "Qwen3.5-LiveTranslate: From Sound to Sight, From Word to Right" }, { "disposition": "included", "id": "candidate_qwen_20260520_qwen3_7", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260520_qwen3_7" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "source_id": "source_candidate_qwen_20260520_qwen3_7", "title": "Qwen3.7: The Agent Frontier" }, { "disposition": "excluded", "id": "candidate_qwen_20260529_qwenvla", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260529_qwenvla" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-29", "reason_code": "excluded_robotics", "source_id": "source_candidate_qwen_20260529_qwenvla", "title": "Qwen-VLA: From Understanding the World to Acting in It" }, { "disposition": "included", "id": "candidate_qwen_20260601_qwen3_7_plus", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260601_qwen3_7_plus" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "source_id": "source_candidate_qwen_20260601_qwen3_7_plus", "title": "Qwen3.7-Plus: Multimodal Agent Intelligence" }, { "disposition": "excluded", "id": "candidate_qwen_20260616_qwen_robotmanip", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260616_qwen_robotmanip" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-16", "reason_code": "excluded_robotics", "source_id": "source_candidate_qwen_20260616_qwen_robotmanip", "title": "Qwen-RobotManip: Alignment Unlocks Scale for Robotic Manipulation Foundation Models" }, { "disposition": "excluded", "id": "candidate_qwen_20260616_qwen_robotnav", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260616_qwen_robotnav" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-16", "reason_code": "excluded_robotics", "source_id": "source_candidate_qwen_20260616_qwen_robotnav", "title": "Qwen-RobotNav: A Scalable Navigation Model Designed for an Agentic Navigation System" }, { "disposition": "excluded", "id": "candidate_qwen_20260616_qwen_robotsuite", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260616_qwen_robotsuite" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-16", "reason_code": "excluded_robotics", "source_id": "source_candidate_qwen_20260616_qwen_robotsuite", "title": "Qwen-Robot Suite: A Foundation Model Suite for Physical World Intelligence" }, { "disposition": "excluded", "id": "candidate_qwen_20260616_qwen_robotworld", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260616_qwen_robotworld" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-16", "reason_code": "excluded_robotics", "source_id": "source_candidate_qwen_20260616_qwen_robotworld", "title": "Qwen-RobotWorld: Boundless Worlds for Embodied Agents" }, { "disposition": "excluded", "id": "candidate_qwen_20260623_qwen_agentworld", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260623_qwen_agentworld" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-23", "reason_code": "excluded_not_included_model_class", "source_id": "source_candidate_qwen_20260623_qwen_agentworld", "title": "Qwen-AgentWorld: Language World Models for General Agents" }, { "disposition": "excluded", "id": "candidate_qwen_20260721_qwen_image_3_0", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260721_qwen_image_3_0" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "reason_code": "excluded_image_only", "source_id": "source_candidate_qwen_20260721_qwen_image_3_0", "title": "Qwen-Image-3.0: Rich Content, Authentic Details, Deep Knowledge" }, { "disposition": "included", "id": "candidate_qwen_20260803_qwen3_8", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260803_qwen3_8" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "source_id": "source_candidate_qwen_20260803_qwen3_8", "title": "Qwen3.8-Max: A New Bar for Coding and Cowork" }, { "disposition": "included", "id": "candidate_qwen_20260826_qwen3_8_flash_next", "lab_id": "qwen", "provenance": { "record_ids": [ "candidate_qwen_20260826_qwen3_8_flash_next" ], "record_type": "release_candidates", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "source_id": "source_candidate_qwen_20260826_qwen3_8_flash_next", "title": "Qwen3.8-Flash-Next: A New Architecture, Towards Ultimate Cost-Efficiency" } ], "releases": [ { "candidate_id": "candidate_anthropic_20240304_introducing_the_next_generation_of_claude", "coverage_id": "coverage_anthropic_20240304_introducing_the_next_generation_of_claude", "id": "release_anthropic_20240304_introducing_the_next_generation_of_claude", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_family", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20240304_introducing_the_next_generation_of_claude", "name": "Introducing the next generation of Claude", "provenance": { "record_ids": [ "release_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20240621_claude_3_5_sonnet", "coverage_id": "coverage_anthropic_20240621_claude_3_5_sonnet", "id": "release_anthropic_20240621_claude_3_5_sonnet", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20240621_claude_3_5_sonnet", "name": "Claude 3.5 Sonnet", "provenance": { "record_ids": [ "release_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "coverage_id": "coverage_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "id": "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_3_5_family", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "name": "Introducing computer use, a new Claude 3.5 Sonnet, and Claude 3.5 Haiku", "provenance": { "record_ids": [ "release_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "coverage_id": "coverage_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "id": "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet_reasoning", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "name": "Claude 3.7 Sonnet and Claude Code", "provenance": { "record_ids": [ "release_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "use_class": "reasoning" }, { "candidate_id": "candidate_anthropic_20250522_introducing_claude_4", "coverage_id": "coverage_anthropic_20250522_introducing_claude_4", "id": "release_anthropic_20250522_introducing_claude_4", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_4_family", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20250522_introducing_claude_4", "name": "Introducing Claude 4", "provenance": { "record_ids": [ "release_anthropic_20250522_introducing_claude_4" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "use_class": "reasoning" }, { "candidate_id": "candidate_anthropic_20250805_claude_opus_4_1", "coverage_id": "coverage_anthropic_20250805_claude_opus_4_1", "id": "release_anthropic_20250805_claude_opus_4_1", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20250805_claude_opus_4_1", "name": "Claude Opus 4.1", "provenance": { "record_ids": [ "release_anthropic_20250805_claude_opus_4_1" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20250929_introducing_claude_sonnet_4_5", "coverage_id": "coverage_anthropic_20250929_introducing_claude_sonnet_4_5", "id": "release_anthropic_20250929_introducing_claude_sonnet_4_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20250929_introducing_claude_sonnet_4_5", "name": "Introducing Claude Sonnet 4.5", "provenance": { "record_ids": [ "release_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20251015_introducing_claude_haiku_4_5", "coverage_id": "coverage_anthropic_20251015_introducing_claude_haiku_4_5", "id": "release_anthropic_20251015_introducing_claude_haiku_4_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_haiku", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20251015_introducing_claude_haiku_4_5", "name": "Introducing Claude Haiku 4.5", "provenance": { "record_ids": [ "release_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20251124_introducing_claude_opus_4_5", "coverage_id": "coverage_anthropic_20251124_introducing_claude_opus_4_5", "id": "release_anthropic_20251124_introducing_claude_opus_4_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20251124_introducing_claude_opus_4_5", "name": "Introducing Claude Opus 4.5", "provenance": { "record_ids": [ "release_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20260205_introducing_claude_opus_4_6", "coverage_id": "coverage_anthropic_20260205_introducing_claude_opus_4_6", "id": "release_anthropic_20260205_introducing_claude_opus_4_6", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20260205_introducing_claude_opus_4_6", "name": "Introducing Claude Opus 4.6", "provenance": { "record_ids": [ "release_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20260217_introducing_claude_sonnet_4_6", "coverage_id": "coverage_anthropic_20260217_introducing_claude_sonnet_4_6", "id": "release_anthropic_20260217_introducing_claude_sonnet_4_6", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20260217_introducing_claude_sonnet_4_6", "name": "Introducing Claude Sonnet 4.6", "provenance": { "record_ids": [ "release_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20260416_introducing_claude_opus_4_7", "coverage_id": "coverage_anthropic_20260416_introducing_claude_opus_4_7", "id": "release_anthropic_20260416_introducing_claude_opus_4_7", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20260416_introducing_claude_opus_4_7", "name": "Introducing Claude Opus 4.7", "provenance": { "record_ids": [ "release_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20260528_introducing_claude_opus_4_8", "coverage_id": "coverage_anthropic_20260528_introducing_claude_opus_4_8", "id": "release_anthropic_20260528_introducing_claude_opus_4_8", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20260528_introducing_claude_opus_4_8", "name": "Introducing Claude Opus 4.8", "provenance": { "record_ids": [ "release_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "coverage_id": "coverage_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "id": "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_fable_mythos", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "name": "Claude Fable 5 and Claude Mythos 5", "provenance": { "record_ids": [ "release_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20260630_introducing_claude_sonnet_5", "coverage_id": "coverage_anthropic_20260630_introducing_claude_sonnet_5", "id": "release_anthropic_20260630_introducing_claude_sonnet_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_sonnet", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20260630_introducing_claude_sonnet_5", "name": "Introducing Claude Sonnet 5", "provenance": { "record_ids": [ "release_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "use_class": "general" }, { "candidate_id": "candidate_anthropic_20260724_introducing_claude_opus_5", "coverage_id": "coverage_anthropic_20260724_introducing_claude_opus_5", "id": "release_anthropic_20260724_introducing_claude_opus_5", "lab_id": "anthropic", "lineage_id": "lineage_anthropic_claude_opus", "modality_class": "vision_language", "model_id": "model_anthropic_model_anthropic_20260724_introducing_claude_opus_5", "name": "Introducing Claude Opus 5", "provenance": { "record_ids": [ "release_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20240517_deepseek_chat", "coverage_id": "coverage_deepseek_20240517_deepseek_chat", "id": "release_deepseek_20240517_deepseek_chat", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v2_0517", "name": "DeepSeek-V2-0517", "provenance": { "record_ids": [ "release_deepseek_20240517_deepseek_chat" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-17", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20240614_deepseek_coder", "coverage_id": "coverage_deepseek_20240614_deepseek_coder", "id": "release_deepseek_20240614_deepseek_coder", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_coder_v2", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_coder_v2_0614", "name": "DeepSeek-Coder-V2-0614", "provenance": { "record_ids": [ "release_deepseek_20240614_deepseek_coder" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-14", "use_class": "coding" }, { "candidate_id": "candidate_deepseek_20240628_deepseek_chat", "coverage_id": "coverage_deepseek_20240628_deepseek_chat", "id": "release_deepseek_20240628_deepseek_chat", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v2_0628", "name": "DeepSeek-V2-0628", "provenance": { "record_ids": [ "release_deepseek_20240628_deepseek_chat" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20240724_deepseek_coder", "coverage_id": "coverage_deepseek_20240724_deepseek_coder", "id": "release_deepseek_20240724_deepseek_coder", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_coder_v2", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_coder_v2_0724", "name": "DeepSeek-Coder-V2-0724", "provenance": { "record_ids": [ "release_deepseek_20240724_deepseek_coder" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-24", "use_class": "coding" }, { "candidate_id": "candidate_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "coverage_id": "coverage_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "id": "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v2_5", "name": "DeepSeek-V2.5", "provenance": { "record_ids": [ "release_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "use_class": "coding" }, { "candidate_id": "candidate_deepseek_20241120_deepseek_r1_lite_preview", "coverage_id": "coverage_deepseek_20241120_deepseek_r1_lite_preview", "id": "release_deepseek_20241120_deepseek_r1_lite_preview", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_r1_lite_preview", "name": "DeepSeek-R1-Lite-Preview", "provenance": { "record_ids": [ "release_deepseek_20241120_deepseek_r1_lite_preview" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-20", "use_class": "reasoning" }, { "candidate_id": "candidate_deepseek_20241210_deepseek_chat", "coverage_id": "coverage_deepseek_20241210_deepseek_chat", "id": "release_deepseek_20241210_deepseek_chat", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v2_5", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v2_5_1210", "name": "DeepSeek-V2.5-1210", "provenance": { "record_ids": [ "release_deepseek_20241210_deepseek_chat" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "use_class": "coding" }, { "candidate_id": "candidate_deepseek_20241226_deepseek_chat", "coverage_id": "coverage_deepseek_20241226_deepseek_chat", "id": "release_deepseek_20241226_deepseek_chat", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v3", "name": "DeepSeek-V3", "provenance": { "record_ids": [ "release_deepseek_20241226_deepseek_chat" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20250120_deepseek_reasoner", "coverage_id": "coverage_deepseek_20250120_deepseek_reasoner", "id": "release_deepseek_20250120_deepseek_reasoner", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_r1", "name": "DeepSeek-R1", "provenance": { "record_ids": [ "release_deepseek_20250120_deepseek_reasoner" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "use_class": "reasoning" }, { "candidate_id": "candidate_deepseek_20250324_deepseek_chat", "coverage_id": "coverage_deepseek_20250324_deepseek_chat", "id": "release_deepseek_20250324_deepseek_chat", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v3_0324", "name": "DeepSeek-V3-0324", "provenance": { "record_ids": [ "release_deepseek_20250324_deepseek_chat" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20250528_deepseek_reasoner", "coverage_id": "coverage_deepseek_20250528_deepseek_reasoner", "id": "release_deepseek_20250528_deepseek_reasoner", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_r1", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_r1_0528", "name": "DeepSeek-R1-0528", "provenance": { "record_ids": [ "release_deepseek_20250528_deepseek_reasoner" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "use_class": "reasoning" }, { "candidate_id": "candidate_deepseek_20250821_deepseek_v3_1", "coverage_id": "coverage_deepseek_20250821_deepseek_v3_1", "id": "release_deepseek_20250821_deepseek_v3_1", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v3_1", "name": "DeepSeek-V3.1", "provenance": { "record_ids": [ "release_deepseek_20250821_deepseek_v3_1" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20250922_deepseek_v3_1_terminus", "coverage_id": "coverage_deepseek_20250922_deepseek_v3_1_terminus", "id": "release_deepseek_20250922_deepseek_v3_1_terminus", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v3_1_terminus", "name": "DeepSeek-V3.1-Terminus", "provenance": { "record_ids": [ "release_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20250929_deepseek_v3_2_exp", "coverage_id": "coverage_deepseek_20250929_deepseek_v3_2_exp", "id": "release_deepseek_20250929_deepseek_v3_2_exp", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v3_2_exp", "name": "DeepSeek-V3.2-Exp", "provenance": { "record_ids": [ "release_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20251201_deepseek_v3_2", "coverage_id": "coverage_deepseek_20251201_deepseek_v3_2", "id": "release_deepseek_20251201_deepseek_v3_2", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v3_2", "name": "DeepSeek-V3.2", "provenance": { "record_ids": [ "release_deepseek_20251201_deepseek_v3_2" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20251201_deepseek_v3_2_speciale", "coverage_id": "coverage_deepseek_20251201_deepseek_v3_2_speciale", "id": "release_deepseek_20251201_deepseek_v3_2_speciale", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v3_speciale", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v3_2_speciale", "name": "DeepSeek-V3.2-Speciale", "provenance": { "record_ids": [ "release_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "use_class": "reasoning" }, { "candidate_id": "candidate_deepseek_20260424_deepseek_v4", "coverage_id": "coverage_deepseek_20260424_deepseek_v4", "id": "release_deepseek_20260424_deepseek_v4", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v4_family", "name": "DeepSeek-V4 family", "provenance": { "record_ids": [ "release_deepseek_20260424_deepseek_v4" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20260731_deepseek_v4_flash_update", "coverage_id": "coverage_deepseek_20260731_deepseek_v4_flash_update", "id": "release_deepseek_20260731_deepseek_v4_flash_update", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v4_flash_0731", "name": "DeepSeek-V4-Flash-0731", "provenance": { "record_ids": [ "release_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20260813_deepseek_v4_pro_update", "coverage_id": "coverage_deepseek_20260813_deepseek_v4_pro_update", "id": "release_deepseek_20260813_deepseek_v4_pro_update", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4", "modality_class": "text", "model_id": "model_deepseek_model_deepseek_v4_pro_0813", "name": "DeepSeek-V4-Pro-0813", "provenance": { "record_ids": [ "release_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "use_class": "general" }, { "candidate_id": "candidate_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "coverage_id": "coverage_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "id": "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "lab_id": "deepseek", "lineage_id": "lineage_deepseek_deepseek_v4_vision", "modality_class": "vision_language", "model_id": "model_deepseek_model_deepseek_v4_flash_vision_exp", "name": "DeepSeek-V4-Flash-Vision-Exp", "provenance": { "record_ids": [ "release_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "coverage_id": "coverage_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "id": "release_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_ultra_1_0", "name": "The next chapter of our Gemini era", "provenance": { "record_ids": [ "release_google_deepmind_20240208_the_next_chapter_of_our_gemini_era" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-08", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "coverage_id": "coverage_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "id": "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_pro_february_preview", "name": "Our next-generation model: Gemini 1.5", "provenance": { "record_ids": [ "release_google_deepmind_20240215_our_next_generation_model_gemini_1_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "coverage_id": "coverage_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "id": "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "modality_class": "text", "model_id": "model_google_deepmind_google_deepmind_gemma_1_release_family", "name": "Gemma: Introducing new state-of-the-art open models", "provenance": { "record_ids": [ "release_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "coverage_id": "coverage_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "id": "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_flash_and_pro_update", "name": "Gemini breaks new ground: a faster model, longer context and AI agents", "provenance": { "record_ids": [ "release_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "coverage_id": "coverage_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "id": "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_1_5_production_update", "name": "Updated production-ready Gemini models, reduced 1.5 Pro pricing, increased rate limits, and more", "provenance": { "record_ids": [ "release_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "coverage_id": "coverage_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "id": "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_experimental", "name": "Introducing Gemini 2.0: our new AI model for the agentic era", "provenance": { "record_ids": [ "release_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "coverage_id": "coverage_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "id": "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_model_family", "name": "Gemini 2.0 is now available to everyone", "provenance": { "record_ids": [ "release_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "coverage_id": "coverage_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "id": "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_family", "name": "Start building with Gemini 2.0 Flash and Flash-Lite", "provenance": { "record_ids": [ "release_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "coverage_id": "coverage_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "id": "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_0_flash_native_image_preview", "name": "Experiment with Gemini 2.0 Flash native image generation", "provenance": { "record_ids": [ "release_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250312_introducing_gemma_3", "coverage_id": "coverage_google_deepmind_20250312_introducing_gemma_3", "id": "release_google_deepmind_20250312_introducing_gemma_3", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemma_3_release_family", "name": "Introducing Gemma 3", "provenance": { "record_ids": [ "release_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "coverage_id": "coverage_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "id": "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_experimental", "name": "Gemini 2.5: Our most intelligent AI model", "provenance": { "record_ids": [ "release_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250417_introducing_gemini_2_5_flash", "coverage_id": "coverage_google_deepmind_20250417_introducing_gemini_2_5_flash", "id": "release_google_deepmind_20250417_introducing_gemini_2_5_flash", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_preview", "name": "Introducing Gemini 2.5 Flash", "provenance": { "record_ids": [ "release_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "coverage_id": "coverage_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "id": "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "name": "Build rich, interactive web apps with an updated Gemini 2.5 Pro", "provenance": { "record_ids": [ "release_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "coverage_id": "coverage_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "id": "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_pro_preview_i_o_edition", "name": "Gemini 2.5 Pro Preview: even better coding performance", "provenance": { "record_ids": [ "release_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "coverage_id": "coverage_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "id": "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemma_3n_preview", "name": "Announcing Gemma 3n preview: Powerful, efficient, mobile-first AI", "provenance": { "record_ids": [ "release_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "coverage_id": "coverage_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "id": "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_i_o_model_family_update", "name": "Gemini 2.5: Our most intelligent models are getting even better", "provenance": { "record_ids": [ "release_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "coverage_id": "coverage_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "id": "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_stable_family_update", "name": "Gemini 2.5: Updates to our family of thinking models", "provenance": { "record_ids": [ "release_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "coverage_id": "coverage_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "id": "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemma_3n", "name": "Introducing Gemma 3n: The developer guide", "provenance": { "record_ids": [ "release_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "coverage_id": "coverage_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "id": "release_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_medgemma_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_medgemma_release_family", "name": "MedGemma: Our most capable open models for health AI development", "provenance": { "record_ids": [ "release_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "coverage_id": "coverage_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "id": "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_t5gemma_text_general", "modality_class": "text", "model_id": "model_google_deepmind_google_deepmind_t5gemma_release_family", "name": "T5Gemma: A new collection of encoder-decoder Gemma models", "provenance": { "record_ids": [ "release_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "coverage_id": "coverage_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "id": "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_flash_lite", "name": "Gemini 2.5 Flash-Lite is now ready for scaled production use", "provenance": { "record_ids": [ "release_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "coverage_id": "coverage_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "id": "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_deep_think", "name": "Try Deep Think in the Gemini app", "provenance": { "record_ids": [ "release_google_deepmind_20250801_try_deep_think_in_the_gemini_app" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "use_class": "reasoning" }, { "candidate_id": "candidate_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "coverage_id": "coverage_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "id": "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_text_general", "modality_class": "text", "model_id": "model_google_deepmind_google_deepmind_gemma_3_270m", "name": "Introducing Gemma 3 270M: The compact model for hyper-efficient AI", "provenance": { "record_ids": [ "release_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "coverage_id": "coverage_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "id": "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_vaultgemma_text_general", "modality_class": "text", "model_id": "model_google_deepmind_google_deepmind_vaultgemma_1b", "name": "VaultGemma: The world's most capable differentially private LLM", "provenance": { "record_ids": [ "release_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "coverage_id": "coverage_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "id": "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_2_5_computer_use", "name": "Introducing the Gemini 2.5 Computer Use model", "provenance": { "record_ids": [ "release_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "use_class": "coding" }, { "candidate_id": "candidate_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "coverage_id": "coverage_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "id": "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_3_pro", "name": "A new era of intelligence with Gemini 3", "provenance": { "record_ids": [ "release_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "coverage_id": "coverage_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "id": "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_3_flash", "name": "Gemini 3 Flash: frontier intelligence built for speed", "provenance": { "record_ids": [ "release_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "coverage_id": "coverage_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "id": "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_deep_think_for_science", "name": "Accelerating Mathematical and Scientific Discovery with Gemini Deep Think", "provenance": { "record_ids": [ "release_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-11", "use_class": "reasoning" }, { "candidate_id": "candidate_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "coverage_id": "coverage_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "id": "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_reasoning_vision_language_reasoning", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_3_deep_think", "name": "Gemini 3 Deep Think: Advancing science, research and engineering", "provenance": { "record_ids": [ "release_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "use_class": "reasoning" }, { "candidate_id": "candidate_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "coverage_id": "coverage_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "id": "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_pro", "name": "Gemini 3.1 Pro: A smarter model for your most complex tasks", "provenance": { "record_ids": [ "release_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "coverage_id": "coverage_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "id": "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_3_1_flash_lite", "name": "Gemini 3.1 Flash-Lite: Built for intelligence at scale", "provenance": { "record_ids": [ "release_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "coverage_id": "coverage_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "id": "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemma_4_release_family", "name": "Gemma 4: Byte for byte, the most capable open models", "provenance": { "record_ids": [ "release_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "coverage_id": "coverage_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "id": "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash", "name": "Gemini 3.5: frontier intelligence with action", "provenance": { "record_ids": [ "release_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "coverage_id": "coverage_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "id": "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemma_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemma_4_12b", "name": "Introducing Gemma 4 12B: a unified, encoder-free multimodal model", "provenance": { "record_ids": [ "release_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "coverage_id": "coverage_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "id": "release_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_diffusiongemma_text_general", "modality_class": "text", "model_id": "model_google_deepmind_google_deepmind_diffusiongemma", "name": "DiffusionGemma: [numeric speed claim redacted] faster text generation", "provenance": { "record_ids": [ "release_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-10", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "coverage_id": "coverage_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "id": "release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_computer_use_vision_language_coding", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_computer_use", "name": "Introducing computer use in Gemini 3.5 Flash", "provenance": { "record_ids": [ "release_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-24", "use_class": "coding" }, { "candidate_id": "candidate_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "coverage_id": "coverage_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "id": "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_cyber_text_coding", "modality_class": "text", "model_id": "model_google_deepmind_google_deepmind_gemini_3_5_flash_cyber", "name": "Introducing Gemini 3.5 Flash Cyber", "provenance": { "record_ids": [ "release_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "use_class": "coding" }, { "candidate_id": "candidate_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "coverage_id": "coverage_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "id": "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_3_6_flash_family", "name": "Introducing Gemini 3.6 Flash, 3.5 Flash-Lite, and 3.5 Flash Cyber", "provenance": { "record_ids": [ "release_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "use_class": "general" }, { "candidate_id": "candidate_google_deepmind_20260813_introducing_gemini_3_7_flash", "coverage_id": "coverage_google_deepmind_20260813_introducing_gemini_3_7_flash", "id": "release_google_deepmind_20260813_introducing_gemini_3_7_flash", "lab_id": "google_deepmind", "lineage_id": "lineage_google_deepmind_google_deepmind_gemini_general_vision_language_general", "modality_class": "vision_language", "model_id": "model_google_deepmind_google_deepmind_gemini_3_7_flash", "name": "Introducing Gemini 3.7 Flash", "provenance": { "record_ids": [ "release_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "use_class": "general" }, { "candidate_id": "candidate_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "coverage_id": "coverage_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "id": "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "modality_class": "text", "model_id": "model_meta_model_meta_llama_3", "name": "Meta Llama 3", "provenance": { "record_ids": [ "release_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "use_class": "general" }, { "candidate_id": "candidate_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "coverage_id": "coverage_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "id": "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "modality_class": "text", "model_id": "model_meta_model_meta_llama_3_1", "name": "Meta Llama 3.1", "provenance": { "record_ids": [ "release_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "use_class": "general" }, { "candidate_id": "candidate_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "coverage_id": "coverage_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "id": "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "modality_class": "vision_language", "model_id": "model_meta_model_meta_llama_3_2", "name": "Meta Llama 3.2 collection", "provenance": { "record_ids": [ "release_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "use_class": "general" }, { "candidate_id": "candidate_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "coverage_id": "coverage_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "id": "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "modality_class": "text", "model_id": "model_meta_model_meta_llama_3_2_quantized", "name": "Quantized Llama 3.2 instruct variants", "provenance": { "record_ids": [ "release_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "use_class": "instruct" }, { "candidate_id": "candidate_meta_20241219_the_future_of_ai_built_with_llama", "coverage_id": "coverage_meta_20241219_the_future_of_ai_built_with_llama", "id": "release_meta_20241219_the_future_of_ai_built_with_llama", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_text", "modality_class": "text", "model_id": "model_meta_model_meta_llama_3_3_70b", "name": "Meta Llama 3.3 70B", "provenance": { "record_ids": [ "release_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "use_class": "general" }, { "candidate_id": "candidate_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "coverage_id": "coverage_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "id": "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_vision", "modality_class": "vision_language", "model_id": "model_meta_model_meta_llama_4", "name": "Meta Llama 4 Scout and Maverick", "provenance": { "record_ids": [ "release_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "use_class": "general" }, { "candidate_id": "candidate_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "coverage_id": "coverage_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "id": "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_llama_instruct", "modality_class": "text", "model_id": "model_meta_model_meta_llama_3_3_8b_preview", "name": "Meta Llama 3.3 8B API preview", "provenance": { "record_ids": [ "release_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "use_class": "instruct" }, { "candidate_id": "candidate_meta_20260408_introducing_muse_spark", "coverage_id": "coverage_meta_20260408_introducing_muse_spark", "id": "release_meta_20260408_introducing_muse_spark", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "modality_class": "vision_language", "model_id": "model_meta_model_meta_muse_spark", "name": "Meta Muse Spark", "provenance": { "record_ids": [ "release_meta_20260408_introducing_muse_spark" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "use_class": "reasoning" }, { "candidate_id": "candidate_meta_20260709_introducing_muse_spark_1_1", "coverage_id": "coverage_meta_20260709_introducing_muse_spark_1_1", "id": "release_meta_20260709_introducing_muse_spark_1_1", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_muse", "modality_class": "vision_language", "model_id": "model_meta_model_meta_muse_spark_1_1", "name": "Meta Muse Spark 1.1", "provenance": { "record_ids": [ "release_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "use_class": "reasoning" }, { "candidate_id": "candidate_meta_20240618_meta_fair_research_new_releases", "coverage_id": "coverage_meta_derived_20240618_meta_chameleon", "id": "release_meta_derived_20240618_meta_chameleon", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_chameleon", "modality_class": "vision_language", "model_id": "model_meta_model_meta_chameleon", "name": "Meta Chameleon", "provenance": { "record_ids": [ "release_meta_derived_20240618_meta_chameleon" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "use_class": "general" }, { "candidate_id": "candidate_meta_20240618_meta_fair_research_new_releases", "coverage_id": "coverage_meta_derived_20240618_multi_token_prediction", "id": "release_meta_derived_20240618_multi_token_prediction", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_multi_token_prediction", "modality_class": "text", "model_id": "model_meta_model_meta_multi_token_prediction", "name": "Multi-Token Prediction checkpoints", "provenance": { "record_ids": [ "release_meta_derived_20240618_multi_token_prediction" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "use_class": "coding" }, { "candidate_id": "candidate_meta_20241018_meta_fair_research_bundle", "coverage_id": "coverage_meta_derived_20241018_code_llama_checkpoints", "id": "release_meta_derived_20241018_code_llama_checkpoints", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_code", "modality_class": "text", "model_id": "model_meta_model_meta_layerskip_code_llama", "name": "LayerSkip Code Llama checkpoints", "provenance": { "record_ids": [ "release_meta_derived_20241018_code_llama_checkpoints" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "use_class": "coding" }, { "candidate_id": "candidate_meta_20241018_meta_fair_research_bundle", "coverage_id": "coverage_meta_derived_20241018_layer_skip_llama_checkpoints", "id": "release_meta_derived_20241018_layer_skip_llama_checkpoints", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_layerskip_llama", "modality_class": "text", "model_id": "model_meta_model_meta_layerskip_llama", "name": "LayerSkip Llama checkpoints", "provenance": { "record_ids": [ "release_meta_derived_20241018_layer_skip_llama_checkpoints" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "use_class": "general" }, { "candidate_id": "candidate_meta_20241212_meta_fair_agents_architecture_bundle", "coverage_id": "coverage_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "id": "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_dynamic_blt", "modality_class": "text", "model_id": "model_meta_model_meta_dynamic_blt", "name": "Meta Dynamic Byte Latent Transformer", "provenance": { "record_ids": [ "release_meta_derived_20241212_meta_dynamic_byte_latent_transformer" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "use_class": "general" }, { "candidate_id": "candidate_meta_20241212_meta_fair_agents_architecture_bundle", "coverage_id": "coverage_meta_derived_20241212_meta_large_concept_models", "id": "release_meta_derived_20241212_meta_large_concept_models", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_large_concept_models", "modality_class": "text", "model_id": "model_meta_model_meta_large_concept_models", "name": "Meta Large Concept Models", "provenance": { "record_ids": [ "release_meta_derived_20241212_meta_large_concept_models" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "use_class": "general" }, { "candidate_id": "candidate_meta_20250417_perception_localization_reasoning_bundle", "coverage_id": "coverage_meta_derived_20250417_meta_perception_language_model", "id": "release_meta_derived_20250417_meta_perception_language_model", "lab_id": "meta", "lineage_id": "lineage_meta_lineage_meta_plm", "modality_class": "vision_language", "model_id": "model_meta_model_meta_plm", "name": "Meta Perception Language Model", "provenance": { "record_ids": [ "release_meta_derived_20250417_meta_perception_language_model" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "use_class": "general" }, { "candidate_id": "candidate_openai_20240125_new_embedding_models_and_api_updates", "coverage_id": "coverage_openai_20240125_new_embedding_models_and_api_updates", "id": "release_openai_20240125_new_embedding_models_and_api_updates", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "modality_class": "text", "model_id": "model_openai_model_openai_20240125_new_embedding_models_and_api_updates", "name": "GPT-4 Turbo dated checkpoints", "provenance": { "record_ids": [ "release_openai_20240125_new_embedding_models_and_api_updates" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "use_class": "general" }, { "candidate_id": "candidate_openai_20240513_hello_gpt_4o", "coverage_id": "coverage_openai_20240513_hello_gpt_4o", "id": "release_openai_20240513_hello_gpt_4o", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20240513_hello_gpt_4o", "name": "GPT-4o", "provenance": { "record_ids": [ "release_openai_20240513_hello_gpt_4o" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-13", "use_class": "general" }, { "candidate_id": "candidate_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "coverage_id": "coverage_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "id": "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4o_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "name": "GPT-4o mini", "provenance": { "record_ids": [ "release_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "use_class": "general" }, { "candidate_id": "candidate_openai_20240912_introducing_openai_o1_preview", "coverage_id": "coverage_openai_20240912_introducing_openai_o1_preview", "id": "release_openai_20240912_introducing_openai_o1_preview", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "modality_class": "text", "model_id": "model_openai_model_openai_20240912_introducing_openai_o1_preview", "name": "OpenAI o1-preview", "provenance": { "record_ids": [ "release_openai_20240912_introducing_openai_o1_preview" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "use_class": "reasoning" }, { "candidate_id": "candidate_openai_20240912_openai_o1_mini", "coverage_id": "coverage_openai_20240912_openai_o1_mini", "id": "release_openai_20240912_openai_o1_mini", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning", "modality_class": "text", "model_id": "model_openai_model_openai_20240912_openai_o1_mini", "name": "OpenAI o1-mini", "provenance": { "record_ids": [ "release_openai_20240912_openai_o1_mini" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "use_class": "reasoning" }, { "candidate_id": "candidate_openai_20241217_openai_o1_and_new_tools_for_developers", "coverage_id": "coverage_openai_20241217_openai_o1_and_new_tools_for_developers", "id": "release_openai_20241217_openai_o1_and_new_tools_for_developers", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o1_reasoning_vision", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20241217_openai_o1_and_new_tools_for_developers", "name": "OpenAI o1 dated snapshot", "provenance": { "record_ids": [ "release_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "use_class": "reasoning" }, { "candidate_id": "candidate_openai_20250123_computer_using_agent", "coverage_id": "coverage_openai_20250123_computer_using_agent", "id": "release_openai_20250123_computer_using_agent", "lab_id": "openai", "lineage_id": "lineage_openai_computer_using_agent", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20250123_computer_using_agent", "name": "Computer-Using Agent", "provenance": { "record_ids": [ "release_openai_20250123_computer_using_agent" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "use_class": "general" }, { "candidate_id": "candidate_openai_20250131_openai_o3_mini", "coverage_id": "coverage_openai_20250131_openai_o3_mini", "id": "release_openai_20250131_openai_o3_mini", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_reasoning_text", "modality_class": "text", "model_id": "model_openai_model_openai_20250131_openai_o3_mini", "name": "OpenAI o3-mini", "provenance": { "record_ids": [ "release_openai_20250131_openai_o3_mini" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "use_class": "reasoning" }, { "candidate_id": "candidate_openai_20250227_introducing_gpt_4_5", "coverage_id": "coverage_openai_20250227_introducing_gpt_4_5", "id": "release_openai_20250227_introducing_gpt_4_5", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_general_text", "modality_class": "text", "model_id": "model_openai_model_openai_20250227_introducing_gpt_4_5", "name": "GPT-4.5", "provenance": { "record_ids": [ "release_openai_20250227_introducing_gpt_4_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "use_class": "general" }, { "candidate_id": "candidate_openai_20250414_introducing_gpt_4_1_in_the_api", "coverage_id": "coverage_openai_20250414_introducing_gpt_4_1_in_the_api", "id": "release_openai_20250414_introducing_gpt_4_1_in_the_api", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_4_1_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20250414_introducing_gpt_4_1_in_the_api", "name": "GPT-4.1 family", "provenance": { "record_ids": [ "release_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "use_class": "general" }, { "candidate_id": "candidate_openai_20250416_introducing_openai_o3_and_o4_mini", "coverage_id": "coverage_openai_20250416_introducing_openai_o3_and_o4_mini", "id": "release_openai_20250416_introducing_openai_o3_and_o4_mini", "lab_id": "openai", "lineage_id": "lineage_openai_openai_o3_o4_reasoning_vision", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20250416_introducing_openai_o3_and_o4_mini", "name": "OpenAI o3 and o4-mini", "provenance": { "record_ids": [ "release_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "use_class": "reasoning" }, { "candidate_id": "candidate_openai_20250516_introducing_codex", "coverage_id": "coverage_openai_20250516_introducing_codex", "id": "release_openai_20250516_introducing_codex", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "modality_class": "text", "model_id": "model_openai_model_openai_20250516_introducing_codex", "name": "Codex research preview", "provenance": { "record_ids": [ "release_openai_20250516_introducing_codex" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-16", "use_class": "coding" }, { "candidate_id": "candidate_openai_20250805_introducing_gpt_oss", "coverage_id": "coverage_openai_20250805_introducing_gpt_oss", "id": "release_openai_20250805_introducing_gpt_oss", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_oss_reasoning", "modality_class": "text", "model_id": "model_openai_model_openai_20250805_introducing_gpt_oss", "name": "gpt-oss family", "provenance": { "record_ids": [ "release_openai_20250805_introducing_gpt_oss" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "use_class": "reasoning" }, { "candidate_id": "candidate_openai_20250807_introducing_gpt_5", "coverage_id": "coverage_openai_20250807_introducing_gpt_5", "id": "release_openai_20250807_introducing_gpt_5", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20250807_introducing_gpt_5", "name": "GPT-5 family", "provenance": { "record_ids": [ "release_openai_20250807_introducing_gpt_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "use_class": "general" }, { "candidate_id": "candidate_openai_20250915_introducing_upgrades_to_codex", "coverage_id": "coverage_openai_20250915_introducing_upgrades_to_codex", "id": "release_openai_20250915_introducing_upgrades_to_codex", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20250915_introducing_upgrades_to_codex", "name": "GPT-5-Codex", "provenance": { "record_ids": [ "release_openai_20250915_introducing_upgrades_to_codex" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "use_class": "coding" }, { "candidate_id": "candidate_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "coverage_id": "coverage_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "id": "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "name": "GPT-5.1 family", "provenance": { "record_ids": [ "release_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "use_class": "general" }, { "candidate_id": "candidate_openai_20251119_building_more_with_gpt_5_1_codex_max", "coverage_id": "coverage_openai_20251119_building_more_with_gpt_5_1_codex_max", "id": "release_openai_20251119_building_more_with_gpt_5_1_codex_max", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20251119_building_more_with_gpt_5_1_codex_max", "name": "GPT-5.1-Codex-Max", "provenance": { "record_ids": [ "release_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "use_class": "coding" }, { "candidate_id": "candidate_openai_20251211_introducing_gpt_5_2", "coverage_id": "coverage_openai_20251211_introducing_gpt_5_2", "id": "release_openai_20251211_introducing_gpt_5_2", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20251211_introducing_gpt_5_2", "name": "GPT-5.2 family", "provenance": { "record_ids": [ "release_openai_20251211_introducing_gpt_5_2" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "use_class": "general" }, { "candidate_id": "candidate_openai_20251218_introducing_gpt_5_2_codex", "coverage_id": "coverage_openai_20251218_introducing_gpt_5_2_codex", "id": "release_openai_20251218_introducing_gpt_5_2_codex", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20251218_introducing_gpt_5_2_codex", "name": "GPT-5.2-Codex", "provenance": { "record_ids": [ "release_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "use_class": "coding" }, { "candidate_id": "candidate_openai_20260205_introducing_gpt_5_3_codex", "coverage_id": "coverage_openai_20260205_introducing_gpt_5_3_codex", "id": "release_openai_20260205_introducing_gpt_5_3_codex", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_vision", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260205_introducing_gpt_5_3_codex", "name": "GPT-5.3-Codex", "provenance": { "record_ids": [ "release_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "use_class": "coding" }, { "candidate_id": "candidate_openai_20260212_introducing_gpt_5_3_codex_spark", "coverage_id": "coverage_openai_20260212_introducing_gpt_5_3_codex_spark", "id": "release_openai_20260212_introducing_gpt_5_3_codex_spark", "lab_id": "openai", "lineage_id": "lineage_openai_openai_codex_text", "modality_class": "text", "model_id": "model_openai_model_openai_20260212_introducing_gpt_5_3_codex_spark", "name": "GPT-5.3-Codex-Spark", "provenance": { "record_ids": [ "release_openai_20260212_introducing_gpt_5_3_codex_spark" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "use_class": "coding" }, { "candidate_id": "candidate_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "coverage_id": "coverage_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "id": "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "name": "GPT-5.3 Instant", "provenance": { "record_ids": [ "release_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "use_class": "general" }, { "candidate_id": "candidate_openai_20260305_introducing_gpt_5_4", "coverage_id": "coverage_openai_20260305_introducing_gpt_5_4", "id": "release_openai_20260305_introducing_gpt_5_4", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260305_introducing_gpt_5_4", "name": "GPT-5.4 family", "provenance": { "record_ids": [ "release_openai_20260305_introducing_gpt_5_4" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "use_class": "general" }, { "candidate_id": "candidate_openai_20260317_introducing_gpt_5_4_mini_and_nano", "coverage_id": "coverage_openai_20260317_introducing_gpt_5_4_mini_and_nano", "id": "release_openai_20260317_introducing_gpt_5_4_mini_and_nano", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260317_introducing_gpt_5_4_mini_and_nano", "name": "GPT-5.4 mini and nano", "provenance": { "record_ids": [ "release_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "use_class": "general" }, { "candidate_id": "candidate_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "coverage_id": "coverage_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "id": "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "name": "GPT-Rosalind", "provenance": { "record_ids": [ "release_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "use_class": "general" }, { "candidate_id": "candidate_openai_20260423_introducing_gpt_5_5", "coverage_id": "coverage_openai_20260423_introducing_gpt_5_5", "id": "release_openai_20260423_introducing_gpt_5_5", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260423_introducing_gpt_5_5", "name": "GPT-5.5 family", "provenance": { "record_ids": [ "release_openai_20260423_introducing_gpt_5_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "use_class": "general" }, { "candidate_id": "candidate_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "coverage_id": "coverage_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "id": "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "name": "GPT-5.5 Instant", "provenance": { "record_ids": [ "release_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "use_class": "general" }, { "candidate_id": "candidate_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "coverage_id": "coverage_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "id": "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_rosalind", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "name": "GPT-Rosalind updated model", "provenance": { "record_ids": [ "release_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "use_class": "general" }, { "candidate_id": "candidate_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "coverage_id": "coverage_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "id": "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "name": "GPT-5.6 Sol preview", "provenance": { "record_ids": [ "release_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "use_class": "general" }, { "candidate_id": "candidate_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "coverage_id": "coverage_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "id": "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "name": "GPT-5.6 family", "provenance": { "record_ids": [ "release_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "use_class": "general" }, { "candidate_id": "candidate_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "coverage_id": "coverage_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "id": "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "name": "GPT-5.6 family price revision", "provenance": { "record_ids": [ "release_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "use_class": "general" }, { "candidate_id": "candidate_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "coverage_id": "coverage_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "id": "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "lab_id": "openai", "lineage_id": "lineage_openai_gpt_5_6_general", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "name": "GPT-5.6 Sol and Luna update", "provenance": { "record_ids": [ "release_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "use_class": "general" }, { "candidate_id": "candidate_openai_20260811_daybreak_models_are_now_available_on_aws", "coverage_id": "coverage_openai_20260811_daybreak_models_are_now_available_on_aws", "id": "release_openai_20260811_daybreak_models_are_now_available_on_aws", "lab_id": "openai", "lineage_id": "lineage_openai_openai_daybreak", "modality_class": "vision_language", "model_id": "model_openai_model_openai_20260811_daybreak_models_are_now_available_on_aws", "name": "Daybreak Blue and Red models", "provenance": { "record_ids": [ "release_openai_20260811_daybreak_models_are_now_available_on_aws" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-11", "use_class": "coding" }, { "candidate_id": "candidate_qwen_20240123_introducing_qwen_qwen", "coverage_id": "coverage_qwen_20240123_introducing_qwen_qwen", "id": "release_qwen_20240123_introducing_qwen_qwen", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240123_introducing_qwen_qwen", "name": "Introducing Qwen", "provenance": { "record_ids": [ "release_qwen_20240123_introducing_qwen_qwen" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240125_introducing_qwen_vl_qwen_vl", "coverage_id": "coverage_qwen_20240125_introducing_qwen_vl_qwen_vl", "id": "release_qwen_20240125_introducing_qwen_vl_qwen_vl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vl_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20240125_introducing_qwen_vl_qwen_vl", "name": "Introducing Qwen-VL", "provenance": { "record_ids": [ "release_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240204_introducing_qwen1_5_qwen1_5", "coverage_id": "coverage_qwen_20240204_introducing_qwen1_5_qwen1_5", "id": "release_qwen_20240204_introducing_qwen1_5_qwen1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240204_introducing_qwen1_5_qwen1_5", "name": "Introducing Qwen1.5", "provenance": { "record_ids": [ "release_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "coverage_id": "coverage_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "id": "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "name": "Qwen1.5-MoE: Matching 7B Model Performance with [activation ratio redacted]", "provenance": { "record_ids": [ "release_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "coverage_id": "coverage_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "id": "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "name": "Qwen1.5-32B: Fitting the Capstone of the Qwen1.5 Language Model Series", "provenance": { "record_ids": [ "release_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "coverage_id": "coverage_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "id": "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_coding", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "name": "Code with CodeQwen1.5", "provenance": { "record_ids": [ "release_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "use_class": "coding" }, { "candidate_id": "candidate_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "coverage_id": "coverage_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "id": "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen1_5_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "name": "Qwen1.5-110B: The First 100B+ Model of the Qwen1.5 Series", "provenance": { "record_ids": [ "release_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "coverage_id": "coverage_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "id": "release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_max_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "name": "Notes on Qwen-Max-0428", "provenance": { "record_ids": [ "release_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-11", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240606_hello_qwen2_qwen2", "coverage_id": "coverage_qwen_20240606_hello_qwen2_qwen2", "id": "release_qwen_20240606_hello_qwen2_qwen2", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240606_hello_qwen2_qwen2", "name": "Hello Qwen2", "provenance": { "record_ids": [ "release_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240807_introducing_qwen2_math_qwen2_math", "coverage_id": "coverage_qwen_20240807_introducing_qwen2_math_qwen2_math", "id": "release_qwen_20240807_introducing_qwen2_math_qwen2_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_text_reasoning", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240807_introducing_qwen2_math_qwen2_math", "name": "Introducing Qwen2-Math", "provenance": { "record_ids": [ "release_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "use_class": "reasoning" }, { "candidate_id": "candidate_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "coverage_id": "coverage_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "id": "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "name": "Qwen2-VL: To See the World More Clearly", "provenance": { "record_ids": [ "release_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "coverage_id": "coverage_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "id": "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "name": "Qwen2.5: A Party of Foundation Models!", "provenance": { "record_ids": [ "release_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "coverage_id": "coverage_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "id": "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "name": "Qwen2.5-Coder: Code More, Learn More!", "provenance": { "record_ids": [ "release_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "use_class": "coding" }, { "candidate_id": "candidate_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "coverage_id": "coverage_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "id": "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "name": "Qwen2.5-LLM: Extending the boundary of LLMs", "provenance": { "record_ids": [ "release_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "use_class": "general" }, { "candidate_id": "candidate_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "coverage_id": "coverage_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "id": "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "modality_class": "text", "model_id": "model_qwen_model_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "name": "Qwen2.5-Math: The world's leading open-sourced mathematical LLMs", "provenance": { "record_ids": [ "release_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "use_class": "reasoning" }, { "candidate_id": "candidate_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "coverage_id": "coverage_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "id": "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_coding", "modality_class": "text", "model_id": "model_qwen_model_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "name": "Qwen2.5-Coder Series: Powerful, Diverse, Practical.", "provenance": { "record_ids": [ "release_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "use_class": "coding" }, { "candidate_id": "candidate_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "coverage_id": "coverage_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "id": "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "name": "Extending the Context Length to 1M Tokens!", "provenance": { "record_ids": [ "release_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "use_class": "general" }, { "candidate_id": "candidate_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "coverage_id": "coverage_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "id": "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "modality_class": "text", "model_id": "model_qwen_model_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "name": "QwQ: Reflect Deeply on the Boundaries of the Unknown", "provenance": { "record_ids": [ "release_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "use_class": "reasoning" }, { "candidate_id": "candidate_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "coverage_id": "coverage_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "id": "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "name": "QVQ: To See the World with Wisdom", "provenance": { "record_ids": [ "release_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "use_class": "reasoning" }, { "candidate_id": "candidate_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "coverage_id": "coverage_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "id": "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "name": "Qwen2.5-1M: Deploy Your Own Qwen with Context Length up to 1M Tokens", "provenance": { "record_ids": [ "release_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "use_class": "general" }, { "candidate_id": "candidate_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "coverage_id": "coverage_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "id": "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "name": "Qwen2.5 VL! Qwen2.5 VL! Qwen2.5 VL!", "provenance": { "record_ids": [ "release_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "use_class": "general" }, { "candidate_id": "candidate_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "coverage_id": "coverage_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "id": "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "name": "Qwen2.5-Max: Exploring the Intelligence of Large-scale MoE Model", "provenance": { "record_ids": [ "release_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "use_class": "general" }, { "candidate_id": "candidate_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "coverage_id": "coverage_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "id": "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "modality_class": "text", "model_id": "model_qwen_model_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "name": "... QwQ-Max-Preview", "provenance": { "record_ids": [ "release_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "use_class": "reasoning" }, { "candidate_id": "candidate_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "coverage_id": "coverage_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "id": "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_text_reasoning", "modality_class": "text", "model_id": "model_qwen_model_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "name": "QwQ-32B: Embracing the Power of Reinforcement Learning", "provenance": { "record_ids": [ "release_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "use_class": "reasoning" }, { "candidate_id": "candidate_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "coverage_id": "coverage_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "id": "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "name": "Qwen2.5-VL-32B: Smarter and Lighter", "provenance": { "record_ids": [ "release_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "use_class": "general" }, { "candidate_id": "candidate_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "coverage_id": "coverage_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "id": "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "name": "Qwen2.5 Omni: See, Hear, Talk, Write, Do It All!", "provenance": { "record_ids": [ "release_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "use_class": "general" }, { "candidate_id": "candidate_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "coverage_id": "coverage_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "id": "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen2_5_vision_language_reasoning", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "name": "QVQ-Max: Think with Evidence", "provenance": { "record_ids": [ "release_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-27", "use_class": "reasoning" }, { "candidate_id": "candidate_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "coverage_id": "coverage_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "id": "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "name": "Qwen3: Think Deeper, Act Faster", "provenance": { "record_ids": [ "release_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "use_class": "general" }, { "candidate_id": "candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "coverage_id": "coverage_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "id": "release_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen_vlo_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "name": "Qwen VLo: From \"Understanding\" the World to \"Depicting\" It", "provenance": { "record_ids": [ "release_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "use_class": "general" }, { "candidate_id": "candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "coverage_id": "coverage_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "id": "release_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "modality_class": "text", "model_id": "model_qwen_model_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "name": "Qwen3-Coder: Agentic Coding in the World", "provenance": { "record_ids": [ "release_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "use_class": "coding" }, { "candidate_id": "candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "coverage_id": "coverage_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "id": "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "name": "Qwen3-Next: Towards Ultimate Training & Inference Efficiency", "provenance": { "record_ids": [ "release_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "use_class": "general" }, { "candidate_id": "candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "coverage_id": "coverage_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "id": "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "name": "Qwen3-Omni: Natively Omni-Modal Foundation Models!", "provenance": { "record_ids": [ "release_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "use_class": "general" }, { "candidate_id": "candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "coverage_id": "coverage_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "id": "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "name": "Qwen3-VL: Sharper Vision, Deeper Thought, Broader Action", "provenance": { "record_ids": [ "release_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "use_class": "general" }, { "candidate_id": "candidate_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "coverage_id": "coverage_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "id": "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "name": "Qwen3-Max: Just Scale it", "provenance": { "record_ids": [ "release_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "use_class": "general" }, { "candidate_id": "candidate_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "coverage_id": "coverage_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "id": "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "name": "Qwen3-Omni-Flash-2025-12-01：Hear You. See You. Follow Smarter!", "provenance": { "record_ids": [ "release_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260126_qwen3_max_thinking", "coverage_id": "coverage_qwen_20260126_qwen3_max_thinking", "id": "release_qwen_20260126_qwen3_max_thinking", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_reasoning", "modality_class": "text", "model_id": "model_qwen_model_qwen_20260126_qwen3_max_thinking", "name": "Pushing Qwen3-Max-Thinking Beyond its Limits", "provenance": { "record_ids": [ "release_qwen_20260126_qwen3_max_thinking" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "use_class": "reasoning" }, { "candidate_id": "candidate_qwen_20260203_qwen3_coder_next", "coverage_id": "coverage_qwen_20260203_qwen3_coder_next", "id": "release_qwen_20260203_qwen3_coder_next", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_text_coding", "modality_class": "text", "model_id": "model_qwen_model_qwen_20260203_qwen3_coder_next", "name": "Qwen3-Coder-Next: Pushing Small Hybrid Models on Agentic Coding", "provenance": { "record_ids": [ "release_qwen_20260203_qwen3_coder_next" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "use_class": "coding" }, { "candidate_id": "candidate_qwen_20260216_qwen3_5", "coverage_id": "coverage_qwen_20260216_qwen3_5", "id": "release_qwen_20260216_qwen3_5", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20260216_qwen3_5", "name": "Qwen3.5: Towards Native Multimodal Agents", "provenance": { "record_ids": [ "release_qwen_20260216_qwen3_5" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260319_qwen3_5_max_preview", "coverage_id": "coverage_qwen_20260319_qwen3_5_max_preview", "id": "release_qwen_20260319_qwen3_5_max_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20260319_qwen3_5_max_preview", "name": "Qwen3.5-Max-Preview Now Available on Arena", "provenance": { "record_ids": [ "release_qwen_20260319_qwen3_5_max_preview" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-19", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260330_qwen3_5_omni", "coverage_id": "coverage_qwen_20260330_qwen3_5_omni", "id": "release_qwen_20260330_qwen3_5_omni", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_5_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20260330_qwen3_5_omni", "name": "Qwen3.5-Omni: Scaling Up, Toward Native Omni-Modal AGI", "provenance": { "record_ids": [ "release_qwen_20260330_qwen3_5_omni" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260402_qwen3_6", "coverage_id": "coverage_qwen_20260402_qwen3_6", "id": "release_qwen_20260402_qwen3_6", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20260402_qwen3_6", "name": "Qwen3.6-Plus: Towards Real World Agents", "provenance": { "record_ids": [ "release_qwen_20260402_qwen3_6" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260415_qwen3_6_35b_a3b", "coverage_id": "coverage_qwen_20260415_qwen3_6_35b_a3b", "id": "release_qwen_20260415_qwen3_6_35b_a3b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20260415_qwen3_6_35b_a3b", "name": "Qwen3.6-35B-A3B: Agentic Coding Power, Now Open to All", "provenance": { "record_ids": [ "release_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260418_qwen3_6_max_preview", "coverage_id": "coverage_qwen_20260418_qwen3_6_max_preview", "id": "release_qwen_20260418_qwen3_6_max_preview", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20260418_qwen3_6_max_preview", "name": "Qwen3.6-Max-Preview: Smarter, Sharper, Still Evolving", "provenance": { "record_ids": [ "release_qwen_20260418_qwen3_6_max_preview" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260422_qwen3_6_27b", "coverage_id": "coverage_qwen_20260422_qwen3_6_27b", "id": "release_qwen_20260422_qwen3_6_27b", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_6_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20260422_qwen3_6_27b", "name": "Qwen3.6-27B: Flagship-Level Coding in a 27B Dense Model", "provenance": { "record_ids": [ "release_qwen_20260422_qwen3_6_27b" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260520_qwen3_7", "coverage_id": "coverage_qwen_20260520_qwen3_7", "id": "release_qwen_20260520_qwen3_7", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20260520_qwen3_7", "name": "Qwen3.7: The Agent Frontier", "provenance": { "record_ids": [ "release_qwen_20260520_qwen3_7" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260601_qwen3_7_plus", "coverage_id": "coverage_qwen_20260601_qwen3_7_plus", "id": "release_qwen_20260601_qwen3_7_plus", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_7_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20260601_qwen3_7_plus", "name": "Qwen3.7-Plus: Multimodal Agent Intelligence", "provenance": { "record_ids": [ "release_qwen_20260601_qwen3_7_plus" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260803_qwen3_8", "coverage_id": "coverage_qwen_20260803_qwen3_8", "id": "release_qwen_20260803_qwen3_8", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_text_general", "modality_class": "text", "model_id": "model_qwen_model_qwen_20260803_qwen3_8", "name": "Qwen3.8-Max: A New Bar for Coding and Cowork", "provenance": { "record_ids": [ "release_qwen_20260803_qwen3_8" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "use_class": "general" }, { "candidate_id": "candidate_qwen_20260826_qwen3_8_flash_next", "coverage_id": "coverage_qwen_20260826_qwen3_8_flash_next", "id": "release_qwen_20260826_qwen3_8_flash_next", "lab_id": "qwen", "lineage_id": "lineage_qwen_lineage_qwen3_8_vision_language_general", "modality_class": "vision_language", "model_id": "model_qwen_model_qwen_20260826_qwen3_8_flash_next", "name": "Qwen3.8-Flash-Next: A New Architecture, Towards Ultimate Cost-Efficiency", "provenance": { "record_ids": [ "release_qwen_20260826_qwen3_8_flash_next" ], "record_type": "releases", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "use_class": "general" } ], "schema_version": "1.0.0", "search_index": [ { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_100q_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_100q_hard", "term": "100q" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_100q_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_100q_hard", "term": "100q hard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_100q_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_100q_hard", "term": "100q hard factual evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_1h_videoqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_1h_videoqa", "term": "1h videoqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench", "term": "2 bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench_airline_corrected" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench_airline_corrected", "term": "2 bench airline corrected" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench_airline_original" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench_airline_original", "term": "2 bench airline original" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench_retail" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench_retail", "term": "2 bench retail" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench_telecom" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench_telecom", "term": "2 bench telecom" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_usamo_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_usamo_2025", "term": "2025 usamo" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_terminal_bench_2_0" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_terminal_bench_2_0", "term": "a launch quotation prints terminal bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aa_briefcase" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aa_briefcase", "term": "aa briefcase" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aa_coding_agent_index" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aa_coding_agent_index", "term": "aa coding agent index" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aa_lcr" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aa_lcr", "term": "aa lcr" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aav_capsid_packaging_prediction" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aav_capsid_packaging_prediction", "term": "aav capsid packaging" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aav_capsid_packaging_prediction" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aav_capsid_packaging_prediction", "term": "aav capsid packaging prediction" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_global_mmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_global_mmlu", "term": "abbreviated as gmmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle", "term": "abbreviated as hle" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_abc_bench_fragment_design" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_abc_bench_fragment_design", "term": "abc bench fd" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_abc_bench_fragment_design" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_abc_bench_fragment_design", "term": "abc bench fragment design" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_abc_bench_liquid_handling" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_abc_bench_liquid_handling", "term": "abc bench lh" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_abc_bench_liquid_handling" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_abc_bench_liquid_handling", "term": "abc bench liquid handling" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_abc_bench_screening_evasion" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_abc_bench_screening_evasion", "term": "abc bench screening evasion" }, { "kind": "category", "provenance": { "record_ids": [ "category_abstract_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_abstract_reasoning", "term": "abstract reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_activitynet_qa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_activitynet_qa", "term": "activitynet qa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_adversarial_squad" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_adversarial_squad", "term": "adv squad" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_adversarial_squad" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_adversarial_squad", "term": "adversarial squad" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agents_last_exam" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agents_last_exam", "term": "agent last exam" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agent_red_teaming" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agent_red_teaming", "term": "agent red teaming" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agent_red_teaming" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agent_red_teaming", "term": "agent red teaming art benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agentdojo" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agentdojo", "term": "agentdojo" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agentharm" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agentharm", "term": "agentharm" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agentharm_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agentharm_verified", "term": "agentharm verified" }, { "kind": "category", "provenance": { "record_ids": [ "category_agentic_and_tool_use" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_agentic_and_tool_use", "term": "agentic and tool use" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agentic_cybermisuse" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agentic_cybermisuse", "term": "agentic cybermisuse" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agentic_misalignment" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agentic_misalignment", "term": "agentic misalignment" }, { "kind": "category", "provenance": { "record_ids": [ "category_agentic_misalignment" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_agentic_misalignment", "term": "agentic misalignment" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agentic_misalignment_suite" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agentic_misalignment_suite", "term": "agentic misalignment evaluation suite" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agentic_misalignment_suite" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agentic_misalignment_suite", "term": "agentic misalignment suite" }, { "kind": "category", "provenance": { "record_ids": [ "category_agentic_search" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_agentic_search", "term": "agentic search" }, { "kind": "category", "provenance": { "record_ids": [ "category_agentic_tool_use" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_agentic_tool_use", "term": "agentic tool use" }, { "kind": "category", "provenance": { "record_ids": [ "category_agents_and_tool_use" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_agents_and_tool_use", "term": "agents and tool use" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agents_last_exam" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agents_last_exam", "term": "agents last exam" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agieval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agieval", "term": "agieval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agieval_english" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agieval_english", "term": "agieval english" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_ai_research_evaluation_suite_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_ai_research_evaluation_suite_1", "term": "ai r d suite 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_ai_research_evaluation_suite_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_ai_research_evaluation_suite_2", "term": "ai r d suite 2" }, { "kind": "category", "provenance": { "record_ids": [ "category_ai_research_and_development" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_ai_research_and_development", "term": "ai research and development" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ai2d" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ai2d", "term": "ai2 diagram" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_c" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_c", "term": "ai2 reasoning challenge" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_c" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_c", "term": "ai2 reasoning challenge challenge set" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ai2d" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ai2d", "term": "ai2d" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ai2d_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ai2d_test", "term": "ai2d test" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aider" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aider", "term": "aider" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aider_edit" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aider_edit", "term": "aider edit" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aider_polyglot" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aider_polyglot", "term": "aider polyglot" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aider_polyglot" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aider_polyglot", "term": "aider s polyglot diff benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aime" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aime", "term": "aime" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aime_2024" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aime_2024", "term": "aime 2024" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aime_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aime_2025", "term": "aime 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aime_2026" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aime_2026", "term": "aime 2026" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aime_year_not_specified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aime_year_not_specified", "term": "aime year not specified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_ai_research_evaluation_suite" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_ai_research_evaluation_suite", "term": "aird" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_metr_data_deduplication" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_metr_data_deduplication", "term": "aird data deduplication" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench_airline_corrected" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench_airline_corrected", "term": "airline corrected" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench_airline_original" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench_airline_original", "term": "airline original" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_airs_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_airs_bench", "term": "airs bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aitz" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aitz", "term": "aitz" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_alfred" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_alfred", "term": "alfred" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_alignbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_alignbench", "term": "alignbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_alignbench_1_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_alignbench_1_1", "term": "alignbench 1 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_alignment_faking" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_alignment_faking", "term": "alignment faking" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_alpacaeval_2_0" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_alpacaeval_2_0", "term": "alpacaeval 2 0" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_alpacaeval_2_0" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_alpacaeval_2_0", "term": "alpacaeval2 0" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_american_mathematics_competition_10" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_american_mathematics_competition_10", "term": "amc 10" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_american_mathematics_competition_12" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_american_mathematics_competition_12", "term": "amc 12" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_amc_2022_23" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_amc_2022_23", "term": "amc 2022 23" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_amc_2023" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_amc_2023", "term": "amc 2023" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_american_mathematics_competition_8" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_american_mathematics_competition_8", "term": "amc 8" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aime_2024" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aime_2024", "term": "american invitational mathematics examination" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aime_2024" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aime_2024", "term": "american invitational mathematics examination 2024" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_american_mathematics_competition_10" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_american_mathematics_competition_10", "term": "american mathematics competition 10" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_american_mathematics_competition_12" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_american_mathematics_competition_12", "term": "american mathematics competition 12" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_american_mathematics_competition_8" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_american_mathematics_competition_8", "term": "american mathematics competition 8" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ammlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ammlu", "term": "ammlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_android_control" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_android_control", "term": "android control" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_androidworld" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_androidworld", "term": "android world" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_androidbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_androidbench", "term": "androidbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_androidworld" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_androidworld", "term": "androidworld" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_art_history" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_art_history", "term": "ap art history" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_biology" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_biology", "term": "ap biology" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_calculus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_calculus", "term": "ap calculus" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_chemistry" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_chemistry", "term": "ap chemistry" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_english_language" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_english_language", "term": "ap english language" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_english_literature" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_english_literature", "term": "ap english literature" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_environmental_science" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_environmental_science", "term": "ap environmental science" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_macroeconomics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_macroeconomics", "term": "ap macroeconomics" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_microeconomics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_microeconomics", "term": "ap microeconomics" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_physics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_physics", "term": "ap physics" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_psychology" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_psychology", "term": "ap psychology" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_statistics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_statistics", "term": "ap statistics" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_united_states_government" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_united_states_government", "term": "ap united states government" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_united_states_history" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_united_states_history", "term": "ap united states history" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ap_world_history" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ap_world_history", "term": "ap world history" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_apex" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_apex", "term": "apex" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_apex_agents" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_apex_agents", "term": "apex agents" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_apex_shortlist" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_apex_shortlist", "term": "apex shortlist" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_apexbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_apexbench", "term": "apexbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_api_bank" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_api_bank", "term": "api bank" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gorilla_api_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gorilla_api_bench", "term": "api bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_apps" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_apps", "term": "apps" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_apps_intro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_apps_intro", "term": "apps intro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_agi_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_agi_1", "term": "arc agi 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_agi_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_agi_2", "term": "arc agi 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_agi_3" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_agi_3", "term": "arc agi 3" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_c" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_c", "term": "arc c" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_c" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_c", "term": "arc challenge" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_easy" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_easy", "term": "arc e" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_easy" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_easy", "term": "arc easy" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_agi_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_agi_2", "term": "arcagi 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arena", "term": "arena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arena_ai_text_leaderboard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arena_ai_text_leaderboard", "term": "arena ai leaderboard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arena_ai_text_leaderboard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arena_ai_text_leaderboard", "term": "arena ai text leaderboard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arena_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arena_hard", "term": "arena hard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arena_hard_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arena_hard_v2", "term": "arena hard v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arena_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arena_hard", "term": "arenahard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arkitscenes" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arkitscenes", "term": "arkitscenes" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_artificial_analysis_coding_agent_index" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_artificial_analysis_coding_agent_index", "term": "artificial analysis coding agent index" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_artificial_analysis_intelligence_index" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_artificial_analysis_intelligence_index", "term": "artificial analysis intelligence index" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arxivmath" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arxivmath", "term": "arxivmath" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_automatic_speech_recognition_from_one_book" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_automatic_speech_recognition_from_one_book", "term": "asrob" }, { "kind": "category", "provenance": { "record_ids": [ "category_audio_and_speech" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_audio_and_speech", "term": "audio and speech" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_automatic_speech_recognition_from_one_book" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_automatic_speech_recognition_from_one_book", "term": "automatic speech recognition from one book" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_automation_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_automation_bench", "term": "automation bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_automation_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_automation_bench", "term": "automationbench" }, { "kind": "category", "provenance": { "record_ids": [ "category_autonomous_cybersecurity" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_autonomous_cybersecurity", "term": "autonomous cybersecurity" }, { "kind": "category", "provenance": { "record_ids": [ "category_autonomy_and_software_engineering" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_autonomy_and_software_engineering", "term": "autonomy and software engineering" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_av_speakerbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_av_speakerbench", "term": "av speakerbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_avut" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_avut", "term": "avut" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_babyvision" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_babyvision", "term": "babyvision" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_big_bench_extra_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_big_bench_extra_hard", "term": "bbeh" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbh" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbh", "term": "bbh" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbq" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbq", "term": "bbq" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbq_ambiguous" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbq_ambiguous", "term": "bbq ambig" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbq_ambiguous" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbq_ambiguous", "term": "bbq ambiguous" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbq_disambiguated" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbq_disambiguated", "term": "bbq disambig" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbq_disambiguated" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbq_disambiguated", "term": "bbq disambiguated" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bc_vl" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bc_vl", "term": "bc vl" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_belebele" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_belebele", "term": "belebele" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bench", "term": "bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tau_bench_telecom" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tau_bench_telecom", "term": "bench telecom" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_benchcad" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_benchcad", "term": "benchcad" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_berkeley_function_calling_leaderboard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_berkeley_function_calling_leaderboard", "term": "berkeley function calling leaderboard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_berkeley_function_calling_leaderboard_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_berkeley_function_calling_leaderboard_v2", "term": "berkeley function calling leaderboard v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_betterchartqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_betterchartqa", "term": "betterchartqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_berkeley_function_calling_leaderboard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_berkeley_function_calling_leaderboard", "term": "bfcl" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_berkeley_function_calling_leaderboard_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_berkeley_function_calling_leaderboard_v2", "term": "bfcl v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bfcl_v3_multiturn" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bfcl_v3_multiturn", "term": "bfcl v3 multiturn" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bfcl_v4" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bfcl_v4", "term": "bfcl v4" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbq" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbq", "term": "bias benchmark for qa bbq" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbq" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbq", "term": "bias benchmark for question answering" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbq" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbq", "term": "bias benchmark for question answering bbq" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_big_bench_extra_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_big_bench_extra_hard", "term": "big bench extra hard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbh" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbh", "term": "big bench hard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_big_finance_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_big_finance_bench", "term": "big finance bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_big_sleep_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_big_sleep_evaluation", "term": "big sleep evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbh" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbh", "term": "bigbench hard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bigcodebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bigcodebench", "term": "bigcodebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_biglaw_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_biglaw_bench", "term": "biglaw bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bigpatent" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bigpatent", "term": "bigpatent" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bioasq" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bioasq", "term": "bioasq" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_biodesign_tools" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_biodesign_tools", "term": "biodesign tools" }, { "kind": "category", "provenance": { "record_ids": [ "category_biological_cloning_workflows" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biological_cloning_workflows", "term": "biological cloning workflows" }, { "kind": "category", "provenance": { "record_ids": [ "category_biological_figure_interpretation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biological_figure_interpretation", "term": "biological figure interpretation" }, { "kind": "category", "provenance": { "record_ids": [ "category_biological_knowledge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biological_knowledge", "term": "biological knowledge" }, { "kind": "category", "provenance": { "record_ids": [ "category_biological_laboratory_skills" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biological_laboratory_skills", "term": "biological laboratory skills" }, { "kind": "category", "provenance": { "record_ids": [ "category_biological_protocol_understanding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biological_protocol_understanding", "term": "biological protocol understanding" }, { "kind": "category", "provenance": { "record_ids": [ "category_biological_risk" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biological_risk", "term": "biological risk" }, { "kind": "category", "provenance": { "record_ids": [ "category_biological_risk_agentic_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biological_risk_agentic_evaluation", "term": "biological risk agentic evaluation" }, { "kind": "category", "provenance": { "record_ids": [ "category_biological_risk_knowledge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biological_risk_knowledge", "term": "biological risk knowledge" }, { "kind": "category", "provenance": { "record_ids": [ "category_biological_sequence_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biological_sequence_reasoning", "term": "biological sequence reasoning" }, { "kind": "category", "provenance": { "record_ids": [ "category_biology_multimodal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biology_multimodal", "term": "biology multimodal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_biolp_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_biolp_bench", "term": "biolp bench" }, { "kind": "category", "provenance": { "record_ids": [ "category_biomedical_question_answering" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_biomedical_question_answering", "term": "biomedical question answering" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_pubmedqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_pubmedqa", "term": "biomedical questions" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_biomysterybench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_biomysterybench", "term": "biomysterybench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_biomysterybench_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_biomysterybench_verified", "term": "biomysterybench verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_biopipelinebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_biopipelinebench", "term": "biopipelinebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_biopipelinebench_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_biopipelinebench_verified", "term": "biopipelinebench verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_biotier_refusal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_biotier_refusal", "term": "biotier refusal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bioweapons_knowledge_questions" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bioweapons_knowledge_questions", "term": "bioweapons knowledge questions" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bioweapons_knowledge_questions" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bioweapons_knowledge_questions", "term": "bioweapons knowledge questions evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bird_sql" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bird_sql", "term": "bird sql" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bixbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bixbench", "term": "bixbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_black_box_rna_sequence_modeling_and_design" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_black_box_rna_sequence_modeling_and_design", "term": "black box rna sequence design" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_black_box_rna_sequence_modeling_and_design" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_black_box_rna_sequence_modeling_and_design", "term": "black box rna sequence modeling and design" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_blackmail_suite" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_blackmail_suite", "term": "blackmail suite" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_blend" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_blend", "term": "blend" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_blink" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_blink", "term": "blink" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_blueprint_bench_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_blueprint_bench_2", "term": "blueprint bench 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bold" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bold", "term": "bold" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_boolq" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_boolq", "term": "boolq" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_browsecomp" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_browsecomp", "term": "browsecomp" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_browsecomp_long_context" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_browsecomp_long_context", "term": "browsecomp long context" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_browsecomp_plus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_browsecomp_plus", "term": "browsecomp plus" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_browsecomp_zh" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_browsecomp_zh", "term": "browsecomp zh" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_browsecomp_zh" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_browsecomp_zh", "term": "browsecompzh" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_c_eval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_c_eval", "term": "c eval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_c_simpleqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_c_simpleqa", "term": "c simpleqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_c3" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_c3", "term": "c3" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_c4" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_c4", "term": "c4" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_capture_the_flag_challenges" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_capture_the_flag_challenges", "term": "capture the flag challenges" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_catastrophic_cyber_misuse" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_catastrophic_cyber_misuse", "term": "catastrophic cyber misuse" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cc_ocr" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cc_ocr", "term": "cc ocr" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cc_ocr_bench_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cc_ocr_bench_v2", "term": "cc ocr bench v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ccpm" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ccpm", "term": "ccpm" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cg_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cg_bench", "term": "cg bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_challenging_refusal_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_challenging_refusal_evaluation", "term": "challenging refusal evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_changemyview" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_changemyview", "term": "changemyview" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_charades_sta" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_charades_sta", "term": "charades sta" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_charadessta" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_charadessta", "term": "charadessta" }, { "kind": "category", "provenance": { "record_ids": [ "category_chart_question_answering" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_chart_question_answering", "term": "chart question answering" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chartqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chartqa", "term": "chart understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chartmimic" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chartmimic", "term": "chartmimic" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chartmuseum" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chartmuseum", "term": "chartmuseum" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chartography" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chartography", "term": "chartography" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chartqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chartqa", "term": "chartqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chartqapro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chartqapro", "term": "chartqapro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_charxiv_descriptive" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_charxiv_descriptive", "term": "charxiv d" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_charxiv_descriptive" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_charxiv_descriptive", "term": "charxiv descriptive" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_charxiv_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_charxiv_reasoning", "term": "charxiv r" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_charxiv_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_charxiv_reasoning", "term": "charxiv reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_charxiv_rq" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_charxiv_rq", "term": "charxiv rq" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lmarena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lmarena", "term": "chatbot arena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chemical_agents_refusal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chemical_agents_refusal", "term": "chemical agents refusal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chemicaldiagramqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chemicaldiagramqa", "term": "chemicaldiagramqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chinese_simpleqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chinese_simpleqa", "term": "chinese simpleqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chrome_production_commit_scanning_pipeline" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chrome_production_commit_scanning_pipeline", "term": "chrome production commit scanning pipeline" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_impossible_tasks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_impossible_tasks", "term": "claude code impossible tasks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_claweval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_claweval", "term": "claweval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_claweval_mm" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_claweval_mm", "term": "claweval mm" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench_cloningscenarios" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench_cloningscenarios", "term": "cloning workflows" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cloningqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cloningqa", "term": "cloningqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cloningscenarios" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cloningscenarios", "term": "cloningscenarios" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cluewsc" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cluewsc", "term": "cluewsc" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cmath" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cmath", "term": "cmath" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cmmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cmmlu", "term": "cmmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cmo_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cmo_2025", "term": "cmo 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cmrc" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cmrc", "term": "cmrc" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cmt_benchmark" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cmt_benchmark", "term": "cmt benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cn_middle_school_2024" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cn_middle_school_2024", "term": "cn middle school 2024" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cnmo_2024" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cnmo_2024", "term": "cnmo 2024" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cnn_dailymail" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cnn_dailymail", "term": "cnn dailymail" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cnn_dailymail" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cnn_dailymail", "term": "cnn dm" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_coco" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_coco", "term": "coco" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_coco_caption" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_coco_caption", "term": "coco caption" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ms_coco_captions" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ms_coco_captions", "term": "coco captions" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_code_arena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_code_arena", "term": "code arena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mbpp" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mbpp", "term": "code generation" }, { "kind": "category", "provenance": { "record_ids": [ "category_code_generation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_code_generation", "term": "code generation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_codecontests" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_codecontests", "term": "codecontests" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_codeforces" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_codeforces", "term": "codeforces" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_codeforces" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_codeforces", "term": "codeforces div1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_codeu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_codeu", "term": "codeu" }, { "kind": "category", "provenance": { "record_ids": [ "category_coding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_coding", "term": "coding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontierbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontierbench", "term": "cognition s frontier coding eval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_collegemath" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_collegemath", "term": "collegemath" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_collegiate_ctfs" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_collegiate_ctfs", "term": "collegiate ctfs" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_collie" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_collie", "term": "collie" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_common_voice" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_common_voice", "term": "common voice" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_common_voice_15" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_common_voice_15", "term": "common voice 15" }, { "kind": "category", "provenance": { "record_ids": [ "category_commonsense_and_science_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_commonsense_and_science_reasoning", "term": "commonsense and science reasoning" }, { "kind": "category", "provenance": { "record_ids": [ "category_commonsense_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_commonsense_reasoning", "term": "commonsense reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_commonsenseqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_commonsenseqa", "term": "commonsenseqa" }, { "kind": "category", "provenance": { "record_ids": [ "category_competition_mathematics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_competition_mathematics", "term": "competition mathematics" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_complexfuncbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_complexfuncbench", "term": "complexfuncbench" }, { "kind": "category", "provenance": { "record_ids": [ "category_computational_biology" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_computational_biology", "term": "computational biology" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_prompt_injection_evaluations" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_prompt_injection_evaluations", "term": "computer use" }, { "kind": "category", "provenance": { "record_ids": [ "category_computer_use" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_computer_use", "term": "computer use" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_computer_use_prompt_injection_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_computer_use_prompt_injection_evaluation", "term": "computer use evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_computer_use_prompt_injection_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_computer_use_prompt_injection_evaluation", "term": "computer use prompt injection evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_copa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_copa", "term": "copa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_corpusqa_1m" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_corpusqa_1m", "term": "corpusqa 1m" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cosmopedia_coherence" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cosmopedia_coherence", "term": "cosmopedia coherence" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cot_control" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cot_control", "term": "cot control" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_countbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_countbench", "term": "countbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_countbench_vqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_countbench_vqa", "term": "countbench vqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_countbench_vqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_countbench_vqa", "term": "countbenchqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_countqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_countqa", "term": "countqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_coursera" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_coursera", "term": "coursera" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_covost2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_covost2", "term": "covost2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_coworkbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_coworkbench", "term": "coworkbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_commonsenseqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_commonsenseqa", "term": "cqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_creative_biology" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_creative_biology", "term": "creative biology" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_creative_biology" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_creative_biology", "term": "creative biology tasks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_creative_writing_v3" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_creative_writing_v3", "term": "creative writing v3" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_critpt" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_critpt", "term": "critpt" }, { "kind": "category", "provenance": { "record_ids": [ "category_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_multilingual", "term": "cross lingual" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_crosscodeeval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_crosscodeeval", "term": "crosscodeeval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_crosscodelongeval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_crosscodelongeval", "term": "crosscodelongeval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_crows_pairs" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_crows_pairs", "term": "crows pairs" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_crux_i" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_crux_i", "term": "crux i" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_crux_o" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_crux_o", "term": "crux o" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cruxeval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cruxeval", "term": "cruxeval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cruxeval_i" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cruxeval_i", "term": "cruxeval i" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cruxeval_o" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cruxeval_o", "term": "cruxeval o" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_csimpleqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_csimpleqa", "term": "csimpleqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_commonsenseqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_commonsenseqa", "term": "csqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_curated_ctfs" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_curated_ctfs", "term": "curated ctfs" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cursorbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cursorbench", "term": "cursorbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cursorbench_3_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cursorbench_3_2", "term": "cursorbench 3 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cute" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cute", "term": "cute" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cv_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cv_bench", "term": "cv bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cybench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cybench", "term": "cybench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cybench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cybench", "term": "cybench test suite" }, { "kind": "category", "provenance": { "record_ids": [ "category_cybersecurity" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_cybersecurity", "term": "cyber" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyber_benign_use" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyber_benign_use", "term": "cyber benign use" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyber_harness_network_challenges" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyber_harness_network_challenges", "term": "cyber harness network challenges" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyber_misuse" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyber_misuse", "term": "cyber misuse" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyber_misuse_chat" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyber_misuse_chat", "term": "cyber misuse chat" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyber_harness_network_challenges" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyber_harness_network_challenges", "term": "cyber ranges" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyberbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyberbench", "term": "cyberbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cybergym" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cybergym", "term": "cybergym" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyberseceval_code_interpreter_abuse" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyberseceval_code_interpreter_abuse", "term": "cyberseceval code interpreter abuse" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyberseceval_insecure_coding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyberseceval_insecure_coding", "term": "cyberseceval insecure coding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyberseceval_text_prompt_injection" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyberseceval_text_prompt_injection", "term": "cyberseceval text prompt injection" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyberseceval_vulnerability_identification" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyberseceval_vulnerability_identification", "term": "cyberseceval vulnerability identification" }, { "kind": "category", "provenance": { "record_ids": [ "category_cybersecurity" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_cybersecurity", "term": "cybersecurity" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cybersecurity_ctf" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cybersecurity_ctf", "term": "cybersecurity ctf" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cyscenariobench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cyscenariobench", "term": "cyscenariobench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_dailyomni" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_dailyomni", "term": "dailyomni" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_deceptionbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_deceptionbench", "term": "deceptionbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_deep_planning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_deep_planning", "term": "deep planning" }, { "kind": "category", "provenance": { "record_ids": [ "category_deep_research" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_deep_research", "term": "deep research" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_deep_research_accuracy_completeness_and_objectivity" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_deep_research_accuracy_completeness_and_objectivity", "term": "deep research accuracy completeness and objectivity" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_deep_research_accuracy_completeness_and_objectivity" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_deep_research_accuracy_completeness_and_objectivity", "term": "deep research accuracy completeness and objectivity draco" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_deepsearchqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_deepsearchqa", "term": "deepsearchqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_deepswe" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_deepswe", "term": "deepswe" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_deepswe_1_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_deepswe_1_1", "term": "deepswe 1 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_deepswe_v1_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_deepswe_v1_1", "term": "deepswe v1 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_dense200" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_dense200", "term": "dense200" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_charxiv_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_charxiv_reasoning", "term": "described as a charxiv chart understanding evaluation suite" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmu_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmu_pro", "term": "described as an improved variant of mmmu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench_airline_corrected" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench_airline_corrected", "term": "described as correcting task setup and grading issues in airline" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_browsecomp_plus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_browsecomp_plus", "term": "described as derived from browsecomp" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_pro", "term": "described as the scale ai variant of swe bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_webarena_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_webarena_verified", "term": "described as the verified release of webarena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_design2code" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_design2code", "term": "design2code" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gpqa_diamond" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gpqa_diamond", "term": "diamond set" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench_seqqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench_seqqa", "term": "dna protein sequence manipulation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_dna_synthesis_screening_evasion" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_dna_synthesis_screening_evasion", "term": "dna synthesis screening evasion" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_securebio_dna_synthesis_screening_evasion" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_securebio_dna_synthesis_screening_evasion", "term": "dna synthesis screening evasion evaluation" }, { "kind": "category", "provenance": { "record_ids": [ "category_document_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_document_reasoning", "term": "document reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_docvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_docvqa", "term": "document understanding" }, { "kind": "category", "provenance": { "record_ids": [ "category_document_visual_question_answering" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_document_visual_question_answering", "term": "document visual question answering" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_docvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_docvqa", "term": "docvqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_docvqa_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_docvqa_test", "term": "docvqa test" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_draco" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_draco", "term": "draco" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_dream_1k" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_dream_1k", "term": "dream 1k" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_drop" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_drop", "term": "drop" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ds_arena_code" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ds_arena_code", "term": "ds arena code" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ds_fim_eval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ds_fim_eval", "term": "ds fim eval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_dsbench_fullstack" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_dsbench_fullstack", "term": "dsbench fullstack" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_dsbench_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_dsbench_hard", "term": "dsbench hard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_dude" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_dude", "term": "dude" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_dynabench_squad" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_dynabench_squad", "term": "dynabench squad" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_dynamath" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_dynamath", "term": "dynamath" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_finance_agent" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_finance_agent", "term": "earlier card prints financeagent" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_easy_medium_qa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_easy_medium_qa", "term": "easy medium qa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_eclektic" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_eclektic", "term": "eclektic" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ego3d_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ego3d_bench", "term": "ego3d bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_egolife_w_mem" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_egolife_w_mem", "term": "egolife w mem" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_egoschema" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_egoschema", "term": "egoschema" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_embspatialbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_embspatialbench", "term": "embspatialbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vibench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vibench", "term": "end to end vibe coding benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_erqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_erqa", "term": "erqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_evalplus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_evalplus", "term": "evalplus" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_humaneval_plus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_humaneval_plus", "term": "evalplus humaneval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_eventhallusion" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_eventhallusion", "term": "eventhallusion" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_aime_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_aime_2025", "term": "expanded as american invitational mathematics examination" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_draco" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_draco", "term": "expanded as deep research accuracy completeness and objectivity" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmu", "term": "expanded as massive multi discipline multimodal understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_mrcr_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_mrcr_v2", "term": "expanded as multi round co reference resolution" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_milu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_milu", "term": "expanded as multi task indic language understanding benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmlu", "term": "expanded as multilingual massive multitask language understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_expert_swe" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_expert_swe", "term": "expert swe" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_exploitbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_exploitbench", "term": "exploit bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_exploitbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_exploitbench", "term": "exploitbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_exploitgym" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_exploitgym", "term": "exploitgym" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_facts_grounding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_facts_grounding", "term": "facts grounding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_facts_parametric" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_facts_parametric", "term": "facts parametric" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_factscore" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_factscore", "term": "factscore" }, { "kind": "category", "provenance": { "record_ids": [ "category_factuality" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_factuality", "term": "factuality" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_false_premise_questions" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_false_premise_questions", "term": "false premise questions" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plm_videobench_fgqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plm_videobench_fgqa", "term": "fgqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_figqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_figqa", "term": "figqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lab_bench_figqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lab_bench_figqa", "term": "figqa is identified as part of language agent biology benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_firefox_147_exploitation_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_firefox_147_exploitation_evaluation", "term": "figure 3 3 3 a calls it the firefox shell exploitation evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench_figqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench_figqa", "term": "figure interpretation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_osworld" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_osworld", "term": "figures 8 9 4 a and 8 9 4 b print osworld verified for the evaluated setup" }, { "kind": "category", "provenance": { "record_ids": [ "category_finance" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_finance", "term": "finance" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_finance_agent" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_finance_agent", "term": "finance agent" }, { "kind": "category", "provenance": { "record_ids": [ "category_finance_agent" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_finance_agent", "term": "finance agent" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_finance_agent_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_finance_agent_v2", "term": "finance agent v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_finance_agent" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_finance_agent", "term": "financeagent" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_financial_crimes" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_financial_crimes", "term": "financial crimes" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_financial_services_benchmark" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_financial_services_benchmark", "term": "financial services benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_firefox_147_exploit_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_firefox_147_exploit_evaluation", "term": "firefox" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_firefox_147_exploit_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_firefox_147_exploit_evaluation", "term": "firefox 147 exploit evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_firefox_147_exploitation_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_firefox_147_exploitation_evaluation", "term": "firefox 147 exploitation evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_firefox_exploit_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_firefox_exploit_evaluation", "term": "firefox exploit evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_firefox_exploit_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_firefox_exploit_evaluation", "term": "firefox exploits" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_fleurs" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_fleurs", "term": "fleurs" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_flickr30k_captions" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_flickr30k_captions", "term": "flickr30k captions" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_flores" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_flores", "term": "flores" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_flores_101" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_flores_101", "term": "flores 101" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_fncall" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_fncall", "term": "fncall" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_fortress" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_fortress", "term": "fortress" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frames" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frames", "term": "frames" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontier_bench_v0_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontier_bench_v0_1", "term": "frontier bench v0 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontier_science_research" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontier_science_research", "term": "frontier science research" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontierbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontierbench", "term": "frontierbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontier_bench_v0_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontier_bench_v0_1", "term": "frontierbench v0 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontiercode" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontiercode", "term": "frontiercode" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontiercode_v1_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontiercode_v1_1", "term": "frontiercode 1 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontiercode" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontiercode", "term": "frontiercode diamond" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontiercode_v1_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontiercode_v1_1", "term": "frontiercode extended" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontiercode_v1_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontiercode_v1_1", "term": "frontiercode main" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontiercode_v1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontiercode_v1", "term": "frontiercode v1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontiercode_v1_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontiercode_v1_1", "term": "frontiercode v1 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontiercode_v2_main" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontiercode_v2_main", "term": "frontiercode v2 main" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontiermath" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontiermath", "term": "frontiermath" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontiermath_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontiermath_v2", "term": "frontiermath v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontierscience_research" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontierscience_research", "term": "frontierscience research" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_frontierswe" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_frontierswe", "term": "frontierswe" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_functional_math" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_functional_math", "term": "functional math" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gaokao" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gaokao", "term": "gaokao" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gaokao_2023_english" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gaokao_2023_english", "term": "gaokao 2023 english" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gaokao_math_cloze" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gaokao_math_cloze", "term": "gaokao math cloze" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gaokao_math_qa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gaokao_math_qa", "term": "gaokao math qa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gdm_mrcr_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gdm_mrcr_v2", "term": "gdm mrcr v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gdm_situational_awareness" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gdm_situational_awareness", "term": "gdm situational awareness" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gdm_stealth" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gdm_stealth", "term": "gdm stealth" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gdp_pdf" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gdp_pdf", "term": "gdp pdf" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gdpval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gdpval", "term": "gdpval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gdpval_aa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gdpval_aa", "term": "gdpval aa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gdpval_aa_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gdpval_aa_v2", "term": "gdpval aa v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_genebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_genebench", "term": "genebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_genebench_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_genebench_pro", "term": "genebench pro" }, { "kind": "category", "provenance": { "record_ids": [ "category_general_capability" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_general_capability", "term": "general capability" }, { "kind": "category", "provenance": { "record_ids": [ "category_general_knowledge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_general_knowledge", "term": "general knowledge" }, { "kind": "category", "provenance": { "record_ids": [ "category_general_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_general_reasoning", "term": "general reasoning" }, { "kind": "category", "provenance": { "record_ids": [ "category_general_reasoning_and_knowledge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_general_reasoning_and_knowledge", "term": "general reasoning and knowledge" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_prompt_injection_evaluations" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_prompt_injection_evaluations", "term": "general tool use" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_global_mmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_global_mmlu", "term": "global mmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_global_mmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_global_mmlu", "term": "global mmlu gmmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_global_mmlu_lite" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_global_mmlu_lite", "term": "global mmlu lite" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_global_piqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_global_piqa", "term": "global piqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gmat_quantitative" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gmat_quantitative", "term": "gmat quantitative" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gmat_verbal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gmat_verbal", "term": "gmat verbal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_global_mmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_global_mmlu", "term": "gmmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_global_mmlu_lite" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_global_mmlu_lite", "term": "gmmlu lite" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gorilla_api_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gorilla_api_bench", "term": "gorilla api bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gpqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gpqa", "term": "gpqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gpqa_biology" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gpqa_biology", "term": "gpqa biology" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gpqa_diamond" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gpqa_diamond", "term": "gpqa diamond" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gpqa_diamond" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gpqa_diamond", "term": "gpqa is expanded as graduate level google proof q a" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gpqa_main" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gpqa_main", "term": "gpqa main" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gsm8k" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gsm8k", "term": "grade school math" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_usamo_2026" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_usamo_2026", "term": "grading follows matharena methodology" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gpqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gpqa", "term": "graduate level google proof q a" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gpqa_diamond" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gpqa_diamond", "term": "graduate level google proof q a diamond" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gpqa_diamond" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gpqa_diamond", "term": "graduate level q a" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_graphwalks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_graphwalks", "term": "graphwalks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_graphwalks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_graphwalks", "term": "graphwalks bfs" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_graphwalks_bfs_256k" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_graphwalks_bfs_256k", "term": "graphwalks bfs 256k" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_graphwalks_parents_256k" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_graphwalks_parents_256k", "term": "graphwalks parents 256k" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_agent_red_teaming" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_agent_red_teaming", "term": "gray swan agent red teaming benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_grayswan_art" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_grayswan_art", "term": "grayswan art" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gre_general_test_analytical_writing" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gre_general_test_analytical_writing", "term": "gre general test analytical writing" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gre_general_test_quantitative_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gre_general_test_quantitative_reasoning", "term": "gre general test quantitative reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gre_general_test_verbal_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gre_general_test_verbal_reasoning", "term": "gre general test verbal reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gre_physics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gre_physics", "term": "gre physics" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gre_general_test_quantitative_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gre_general_test_quantitative_reasoning", "term": "gre quantitative" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gre_general_test_verbal_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gre_general_test_verbal_reasoning", "term": "gre verbal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gre_general_test_analytical_writing" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gre_general_test_analytical_writing", "term": "gre writing" }, { "kind": "category", "provenance": { "record_ids": [ "category_factuality" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_factuality", "term": "grounding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gsm" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gsm", "term": "gsm" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gsm_plus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gsm_plus", "term": "gsm plus" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gsm8k" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gsm8k", "term": "gsm8k" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gtzan" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gtzan", "term": "gtzan" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_project_gutenberg" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_project_gutenberg", "term": "gutenberg" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gym_cards" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gym_cards", "term": "gym cards" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hallbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hallbench", "term": "hallbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hallusionbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hallusionbench", "term": "hallusionbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_verified_hard_subset" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_verified_hard_subset", "term": "hard subset of swe bench verified" }, { "kind": "category", "provenance": { "record_ids": [ "category_health_and_biology" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_health_and_biology", "term": "health and biology" }, { "kind": "category", "provenance": { "record_ids": [ "category_health_and_science" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_health_and_science", "term": "health and science" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_healthadminbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_healthadminbench", "term": "healthadminbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_healthbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_healthbench", "term": "healthbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_healthbench_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_healthbench_hard", "term": "healthbench hard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_healthbench_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_healthbench_pro", "term": "healthbench pro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_healthbench_professional" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_healthbench_professional", "term": "healthbench professional" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_healthbench_professional" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_healthbench_professional", "term": "healthbench professional adjusted" }, { "kind": "category", "provenance": { "record_ids": [ "category_healthcare" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_healthcare", "term": "healthcare" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hebbia_finance_benchmark" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hebbia_finance_benchmark", "term": "hebbia finance benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hebbia_finance_benchmark" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hebbia_finance_benchmark", "term": "hebbia s finance benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hellaswag" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hellaswag", "term": "hellaswag" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hellaswag_noise" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hellaswag_noise", "term": "hellaswag noise" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_math" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_math", "term": "hendrycks math" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hiddenmath" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hiddenmath", "term": "hiddenmath" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_high_school_ctfs" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_high_school_ctfs", "term": "high school ctfs" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hipho" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hipho", "term": "hipho" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle", "term": "hle" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle_calibration" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle_calibration", "term": "hle calibration" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle_verified", "term": "hle verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle_vl_w_tools" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle_vl_w_tools", "term": "hle vl w tools" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle_with_tools" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle_with_tools", "term": "hle with tools" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hmmt_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hmmt_2025", "term": "hmmt 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hmmt_2026_feb" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hmmt_2026_feb", "term": "hmmt 2026 feb" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hmmt_feb_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hmmt_feb_2025", "term": "hmmt feb 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hmmt_february_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hmmt_february_2025", "term": "hmmt february 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hmmt_february_2026" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hmmt_february_2026", "term": "hmmt february 2026" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hmmt_nov_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hmmt_nov_2025", "term": "hmmt nov 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hmmt_november_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hmmt_november_2025", "term": "hmmt november 2025" }, { "kind": "category", "provenance": { "record_ids": [ "category_honesty" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_honesty", "term": "honesty" }, { "kind": "category", "provenance": { "record_ids": [ "category_honesty_and_hallucination" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_honesty_and_hallucination", "term": "honesty and hallucination" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hpct" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hpct", "term": "hpct" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle", "term": "human last exam" }, { "kind": "category", "provenance": { "record_ids": [ "category_human_preference" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_human_preference", "term": "human preference" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_human_sourced_jailbreaks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_human_sourced_jailbreaks", "term": "human sourced jailbreaks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_humaneval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_humaneval", "term": "humaneval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_humaneval_infilling" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_humaneval_infilling", "term": "humaneval infilling" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_humaneval_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_humaneval_multilingual", "term": "humaneval mul" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_humaneval_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_humaneval_multilingual", "term": "humaneval multi" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_humaneval_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_humaneval_multilingual", "term": "humaneval multilingual" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_humaneval_plus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_humaneval_plus", "term": "humaneval plus" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_humaneval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_humaneval", "term": "humaneval python" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle", "term": "humanity s last exam" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle", "term": "humanity s last exam hle" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle", "term": "humanity s last exam with tools" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hypersim" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hypersim", "term": "hypersim" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_international_chemistry_olympiad_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_international_chemistry_olympiad_2025", "term": "icho 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_icpc_wf_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_icpc_wf_2025", "term": "icpc wf 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lab_bench_figqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lab_bench_figqa", "term": "identified as part of language agent biology benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ifeval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ifeval", "term": "if eval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ifbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ifbench", "term": "ifbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ifeval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ifeval", "term": "ifeval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ifeval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ifeval", "term": "ifeval benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_iheval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_iheval", "term": "iheval" }, { "kind": "category", "provenance": { "record_ids": [ "category_image_understanding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_image_understanding", "term": "image understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_imo_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_imo_2025", "term": "imo 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_imo_2026" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_imo_2026", "term": "imo 2026" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_imoanswerbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_imoanswerbench", "term": "imoanswerbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_impossible_tasks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_impossible_tasks", "term": "impossible tasks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_impossiblebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_impossiblebench", "term": "impossiblebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_incalmo_cyber_ranges" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_incalmo_cyber_ranges", "term": "incalmo cyber ranges" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_include" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_include", "term": "include" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_indommlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_indommlu", "term": "indommlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_infinitebench_en_mc" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_infinitebench_en_mc", "term": "infinitebench en mc" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_infinitebench_en_qa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_infinitebench_en_qa", "term": "infinitebench en qa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_infographicsvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_infographicsvqa", "term": "infographicsqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_infographicsvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_infographicsvqa", "term": "infographicsvqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_infographicvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_infographicvqa", "term": "infographicvqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_infographicsvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_infographicsvqa", "term": "infoqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_infographicvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_infographicvqa", "term": "infovqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_infovqa_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_infovqa_test", "term": "infovqa test" }, { "kind": "category", "provenance": { "record_ids": [ "category_instruction_and_preference" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_instruction_and_preference", "term": "instruction and preference" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ifeval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ifeval", "term": "instruction following" }, { "kind": "category", "provenance": { "record_ids": [ "category_instruction_following" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_instruction_following", "term": "instruction following" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ifeval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ifeval", "term": "instruction following evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_instruction_hierarchy_conflicts" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_instruction_hierarchy_conflicts", "term": "instruction hierarchy conflicts" }, { "kind": "category", "provenance": { "record_ids": [ "category_internal_ai_research_autonomy" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_internal_ai_research_autonomy", "term": "internal ai research autonomy" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_ai_research_evaluation_suite" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_ai_research_evaluation_suite", "term": "internal ai research evaluation suite" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_ai_research_evaluation_suite_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_ai_research_evaluation_suite_1", "term": "internal ai research evaluation suite 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_ai_research_evaluation_suite_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_ai_research_evaluation_suite_2", "term": "internal ai research evaluation suite 2" }, { "kind": "category", "provenance": { "record_ids": [ "category_internal_factuality" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_internal_factuality", "term": "internal factuality" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_prompt_injection_evaluations" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_prompt_injection_evaluations", "term": "internal prompt injection evaluations" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_sycophancy" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_sycophancy", "term": "internal sycophancy" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_international_chemistry_olympiad_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_international_chemistry_olympiad_2025", "term": "international chemistry olympiad 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_imo_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_imo_2025", "term": "international mathematical olympiad 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_international_physics_olympiad_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_international_physics_olympiad_2025", "term": "international physics olympiad 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_international_physics_olympiad_2025_theory" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_international_physics_olympiad_2025_theory", "term": "international physics olympiad 2025 theory" }, { "kind": "category", "provenance": { "record_ids": [ "category_interpretability" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_interpretability", "term": "interpretability" }, { "kind": "category", "provenance": { "record_ids": [ "category_interpretability_control" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_interpretability_control", "term": "interpretability control" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wildchat" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wildchat", "term": "inthe wildchat" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ioi_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ioi_2025", "term": "ioi 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_jailbreak_augmented_examples" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_jailbreak_augmented_examples", "term": "jailbreak augmented examples" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_jmmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_jmmlu", "term": "jmmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_jobbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_jobbench", "term": "jobbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_kernel_bench_l3" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_kernel_bench_l3", "term": "kernel bench l3" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_kernel_optimization" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_kernel_optimization", "term": "kernel optimization" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_kernelgen_1p" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_kernelgen_1p", "term": "kernelgen 1p" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_kernels_task" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_kernels_task", "term": "kernels task" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_kespeech" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_kespeech", "term": "kespeech" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_kmmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_kmmlu", "term": "kmmlu" }, { "kind": "category", "provenance": { "record_ids": [ "category_reasoning_and_knowledge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_reasoning_and_knowledge", "term": "knowledge" }, { "kind": "category", "provenance": { "record_ids": [ "category_knowledge_and_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_knowledge_and_reasoning", "term": "knowledge and reasoning" }, { "kind": "category", "provenance": { "record_ids": [ "category_knowledge_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_knowledge_reasoning", "term": "knowledge reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_korbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_korbench", "term": "korbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_l_eval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_l_eval", "term": "l eval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_legal_agent_benchmark" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_legal_agent_benchmark", "term": "lab" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lab_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lab_bench", "term": "lab bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lab_bench_figqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lab_bench_figqa", "term": "lab bench figqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lab_bench_protocolqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lab_bench_protocolqa", "term": "lab bench protocolqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lab_bench_seqqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lab_bench_seqqa", "term": "lab bench seqqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench", "term": "labbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench_cloningscenarios" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench_cloningscenarios", "term": "labbench cloningscenarios" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench", "term": "labbench evaluations" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench_figqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench_figqa", "term": "labbench figqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench_protocolqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench_protocolqa", "term": "labbench protocolqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench_seqqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench_seqqa", "term": "labbench seqqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench", "term": "labbench subset" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench2", "term": "labbench2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labworkbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labworkbench", "term": "labworkbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lab_bench_figqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lab_bench_figqa", "term": "language agent biology benchmark figqa" }, { "kind": "category", "provenance": { "record_ids": [ "category_language_modeling" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_language_modeling", "term": "language modeling" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_latchbio_bioinformatics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_latchbio_bioinformatics", "term": "latchbio bioinformatics" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle", "term": "later cards abbreviate the benchmark as hle" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_finance_agent" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_finance_agent", "term": "later cards print finance agent with a space" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_terminal_bench_2_0" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_terminal_bench_2_0", "term": "launch quotations also print terminal bench and tbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_law_school_admission_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_law_school_admission_test", "term": "law school admission test" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lcfo_short_summary_condition" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lcfo_short_summary_condition", "term": "lcfo five percent condition" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lcfo_long_summary_condition" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lcfo_long_summary_condition", "term": "lcfo long summary condition" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lcfo_medium_summary_condition" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lcfo_medium_summary_condition", "term": "lcfo medium summary condition" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lcfo_short_summary_condition" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lcfo_short_summary_condition", "term": "lcfo short summary condition" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lcfo_medium_summary_condition" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lcfo_medium_summary_condition", "term": "lcfo ten percent condition" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lcfo_long_summary_condition" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lcfo_long_summary_condition", "term": "lcfo twenty percent condition" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_legal_agent_benchmark" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_legal_agent_benchmark", "term": "legal agent benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_librispeech" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_librispeech", "term": "librispeech" }, { "kind": "category", "provenance": { "record_ids": [ "category_life_sciences" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_life_sciences", "term": "life sciences" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lifescibench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lifescibench", "term": "lifescibench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lingoqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lingoqa", "term": "lingoqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_linuxarena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_linuxarena", "term": "linuxarena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_livebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_livebench", "term": "livebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_livebench_coding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_livebench_coding", "term": "livebench coding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_livebench_july_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_livebench_july_2025", "term": "livebench july 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_livecodebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_livecodebench", "term": "livecodebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_livecodebench_base" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_livecodebench_base", "term": "livecodebench base" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_livecodebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_livecodebench", "term": "livecodebench date window variants" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_livecodebench_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_livecodebench_pro", "term": "livecodebench pro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_livecodebench_v5" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_livecodebench_v5", "term": "livecodebench v5" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_livecodebench_v6" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_livecodebench_v6", "term": "livecodebench v6" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_llm_training" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_llm_training", "term": "llm training" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_llm_training_optimization" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_llm_training_optimization", "term": "llm training optimization" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lmarena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lmarena", "term": "lmarena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lmarena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lmarena", "term": "lmsys chatbot arena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lmarena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lmarena", "term": "lmsys leaderboard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_logicvista" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_logicvista", "term": "logicvista" }, { "kind": "category", "provenance": { "record_ids": [ "category_long_context" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_long_context", "term": "long context" }, { "kind": "category", "provenance": { "record_ids": [ "category_long_context_reading_comprehension" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_long_context_reading_comprehension", "term": "long context reading comprehension" }, { "kind": "category", "provenance": { "record_ids": [ "category_long_context_software_engineering" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_long_context_software_engineering", "term": "long context software engineering" }, { "kind": "category", "provenance": { "record_ids": [ "category_long_context_understanding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_long_context_understanding", "term": "long context understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_long_form_virology_task_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_long_form_virology_task_1", "term": "long form virology task 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_long_form_virology_task_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_long_form_virology_task_2", "term": "long form virology task 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_long_form_virology_tasks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_long_form_virology_tasks", "term": "long form virology tasks" }, { "kind": "category", "provenance": { "record_ids": [ "category_long_horizon_agents" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_long_horizon_agents", "term": "long horizon agents" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_longbench_chat" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_longbench_chat", "term": "longbench chat" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_longbench_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_longbench_v2", "term": "longbench v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_longfact" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_longfact", "term": "longfact" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_longvideobench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_longvideobench", "term": "longvideobench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_law_school_admission_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_law_school_admission_test", "term": "lsat" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lv_eval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lv_eval", "term": "lv eval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lvbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lvbench", "term": "lvbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lvbench_w_mem" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lvbench_w_mem", "term": "lvbench w mem" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_m_mmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_m_mmlu", "term": "m mmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_m3exams" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_m3exams", "term": "m3exams" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_machine_translation_from_one_book" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_machine_translation_from_one_book", "term": "machine translation from one book" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_madqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_madqa", "term": "madqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_makemepay" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_makemepay", "term": "makemepay" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_makemesay" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_makemesay", "term": "makemesay" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_many_shot_gsm8k" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_many_shot_gsm8k", "term": "many shot gsm8k" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mask" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mask", "term": "mask" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmu", "term": "massive multi discipline multimodal understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "term": "massive multi discipline multimodal understanding and reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_massive_multitask_language_understanding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_massive_multitask_language_understanding", "term": "massive multitask language understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_math" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_math", "term": "math" }, { "kind": "category", "provenance": { "record_ids": [ "category_mathematics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_mathematics", "term": "math" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_math_500" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_math_500", "term": "math 500" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_math" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_math", "term": "math dataset" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_matharena_apex" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_matharena_apex", "term": "matharena apex" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_math" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_math", "term": "mathematical problem solving" }, { "kind": "category", "provenance": { "record_ids": [ "category_mathematical_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_mathematical_reasoning", "term": "mathematical reasoning" }, { "kind": "category", "provenance": { "record_ids": [ "category_mathematics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_mathematics", "term": "mathematics" }, { "kind": "category", "provenance": { "record_ids": [ "category_mathematics_and_science" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_mathematics_and_science", "term": "mathematics and science" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mathverse" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mathverse", "term": "mathverse" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mathvision" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mathvision", "term": "mathvision" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mathvista" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mathvista", "term": "mathvista" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mathvista_mini" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mathvista_mini", "term": "mathvista mini" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_maxife" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_maxife", "term": "maxife" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mbct" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mbct", "term": "mbct" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multistate_bar_examination" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multistate_bar_examination", "term": "mbe" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mbpp" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mbpp", "term": "mbpp" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mbpp_evalplus_base" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mbpp_evalplus_base", "term": "mbpp evalplus base" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mbpp_plus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mbpp_plus", "term": "mbpp plus" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mceval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mceval", "term": "mceval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcp_atlas" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcp_atlas", "term": "mcp atlas" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcp_prompt_injection_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcp_prompt_injection_evaluation", "term": "mcp evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcp_mark" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcp_mark", "term": "mcp mark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcp_prompt_injection_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcp_prompt_injection_evaluation", "term": "mcp prompt injection evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcp_universe" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcp_universe", "term": "mcp universe" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcpatlas" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcpatlas", "term": "mcpatlas" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcpatlas_public" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcpatlas_public", "term": "mcpatlas public" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mdeval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mdeval", "term": "mdeval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_medcalc_bench_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_medcalc_bench_verified", "term": "medcalc bench verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_medchembench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_medchembench", "term": "medchembench" }, { "kind": "category", "provenance": { "record_ids": [ "category_healthcare" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_healthcare", "term": "medical" }, { "kind": "category", "provenance": { "record_ids": [ "category_medical_knowledge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_medical_knowledge", "term": "medical knowledge" }, { "kind": "category", "provenance": { "record_ids": [ "category_medical_question_answering" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_medical_question_answering", "term": "medical question answering" }, { "kind": "category", "provenance": { "record_ids": [ "category_medical_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_medical_reasoning", "term": "medical reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_medmcqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_medmcqa", "term": "medmcqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_medqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_medqa", "term": "medqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_medxpertqa_mm" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_medxpertqa_mm", "term": "medxpertqa mm" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_medxpertqa_multimodal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_medxpertqa_multimodal", "term": "medxpertqa multimodal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_medxpertqa_text" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_medxpertqa_text", "term": "medxpertqa text" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_megabench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_megabench", "term": "megabench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_meta_attack_automation_framework" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_meta_attack_automation_framework", "term": "meta attack automation framework" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_meta_spear_phishing_benchmark" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_meta_spear_phishing_benchmark", "term": "meta spear phishing benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_metr_data_deduplication" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_metr_data_deduplication", "term": "metr data deduplication" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mgsm" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mgsm", "term": "mgsm" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_milu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_milu", "term": "milu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_minerva_math" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_minerva_math", "term": "minerva math" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_minimal_linuxbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_minimal_linuxbench", "term": "minimal linuxbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_minimax_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_minimax_multilingual", "term": "minimax multilingual" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mir_1k" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mir_1k", "term": "mir 1k" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bbh" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bbh", "term": "mixed evaluations" }, { "kind": "category", "provenance": { "record_ids": [ "category_mixed_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_mixed_reasoning", "term": "mixed reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mle_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mle_bench", "term": "mle bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mls_bench_lite" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mls_bench_lite", "term": "mls bench lite" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mlvu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mlvu", "term": "mlvu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mm_mt_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mm_mt_bench", "term": "mm mt bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmar" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmar", "term": "mmar" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmau" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmau", "term": "mmau" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmbc" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmbc", "term": "mmbc" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmbench", "term": "mmbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmbench_1_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmbench_1_1", "term": "mmbench 1 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmbench_en_1_1_dev" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmbench_en_1_1_dev", "term": "mmbench en 1 1 dev" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmbench_video" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmbench_video", "term": "mmbench video" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mme" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mme", "term": "mme" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mme_videoocr" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mme_videoocr", "term": "mme videoocr" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmlongbench_doc" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmlongbench_doc", "term": "mmlongbench doc" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_massive_multitask_language_understanding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_massive_multitask_language_understanding", "term": "mmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmlu_multi" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmlu_multi", "term": "mmlu multi" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmlu_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmlu_multilingual", "term": "mmlu multilingual" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmlu_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmlu_pro", "term": "mmlu pro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmlu_prox" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmlu_prox", "term": "mmlu prox" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmlu_redux" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmlu_redux", "term": "mmlu redux" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmlu_stem" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmlu_stem", "term": "mmlu stem" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmlu", "term": "mmmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmlu_non_english" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmlu_non_english", "term": "mmmlu non english" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmu", "term": "mmmu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmu_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmu_pro", "term": "mmmu pro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmu_pro_standard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmu_pro_standard", "term": "mmmu pro standard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmu_pro_vision" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmu_pro_vision", "term": "mmmu pro vision" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmu_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmu_pro", "term": "mmmupro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmsearchplus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmsearchplus", "term": "mmsearchplus" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmstar" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmstar", "term": "mmstar" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmsu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmsu", "term": "mmsu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmt_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmt_bench", "term": "mmt bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmvet" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmvet", "term": "mmvet" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmvu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmvu", "term": "mmvu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mobileworld" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mobileworld", "term": "mobileworld" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mask" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mask", "term": "model alignment between statements and knowledge" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mask" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mask", "term": "model alignment between statements and knowledge mask" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_internal_prompt_injection_evaluations" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_internal_prompt_injection_evaluations", "term": "model context protocol mcp" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcp_prompt_injection_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcp_prompt_injection_evaluation", "term": "model context protocol mcp evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mbpp" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mbpp", "term": "mostly basic python problems" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_motionbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_motionbench", "term": "motionbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mrcr_1m" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mrcr_1m", "term": "mrcr 1m" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mrcr_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mrcr_v2", "term": "mrcr v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ms_coco_captions" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ms_coco_captions", "term": "ms coco captions" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mt_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mt_bench", "term": "mt bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_machine_translation_from_one_book" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_machine_translation_from_one_book", "term": "mtob" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mtob_full_book" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mtob_full_book", "term": "mtob full book" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mtob_half_book" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mtob_half_book", "term": "mtob half book" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mtvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mtvqa", "term": "mtvqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mtvqa_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mtvqa_test", "term": "mtvqa test" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_muirbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_muirbench", "term": "muirbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_browsecomp" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_browsecomp", "term": "multi agent" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_browsecomp" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_browsecomp", "term": "multi agent browsecomp" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_programbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_programbench", "term": "multi agent programbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multi_exam" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multi_exam", "term": "multi exam" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multi_factual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multi_factual", "term": "multi factual" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multi_factual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multi_factual", "term": "multi factual evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multi_if" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multi_if", "term": "multi if" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multi_mathematics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multi_mathematics", "term": "multi mathematics" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_nih_multi_needle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_nih_multi_needle", "term": "multi needle" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multi_news" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multi_news", "term": "multi news" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multi_task_indic_language_understanding_benchmark" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multi_task_indic_language_understanding_benchmark", "term": "multi task indic language understanding benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multi_translation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multi_translation", "term": "multi translation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multi_understanding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multi_understanding", "term": "multi understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multichallenge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multichallenge", "term": "multichallenge" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multiif" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multiif", "term": "multiif" }, { "kind": "category", "provenance": { "record_ids": [ "category_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_multilingual", "term": "multilingual" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mgsm" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mgsm", "term": "multilingual grade school math" }, { "kind": "category", "provenance": { "record_ids": [ "category_multilingual_knowledge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_multilingual_knowledge", "term": "multilingual knowledge" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mgsm" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mgsm", "term": "multilingual math" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mgsm" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mgsm", "term": "multilingual math mgsm" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmlu", "term": "multilingual mmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmlu", "term": "multilingual mmlu evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multiloko" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multiloko", "term": "multiloko" }, { "kind": "category", "provenance": { "record_ids": [ "category_multimodal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_multimodal", "term": "multimodal" }, { "kind": "category", "provenance": { "record_ids": [ "category_multimodal_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_multimodal_reasoning", "term": "multimodal reasoning" }, { "kind": "category", "provenance": { "record_ids": [ "category_multimodal_software_engineering" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_multimodal_software_engineering", "term": "multimodal software engineering" }, { "kind": "category", "provenance": { "record_ids": [ "category_multimodal_understanding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_multimodal_understanding", "term": "multimodal understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_secure_bio_lab_assistance_benchmark_multimodal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_secure_bio_lab_assistance_benchmark_multimodal", "term": "multimodal virology" }, { "kind": "category", "provenance": { "record_ids": [ "category_multimodal_virology_knowledge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_multimodal_virology_knowledge", "term": "multimodal virology knowledge" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multipl_e" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multipl_e", "term": "multipl e" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_secure_bio_lab_assistance_benchmark_multimodal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_secure_bio_lab_assistance_benchmark_multimodal", "term": "multiple select variant" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multistate_bar_examination" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multistate_bar_examination", "term": "multistate bar exam" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_multistate_bar_examination" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_multistate_bar_examination", "term": "multistate bar examination" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mvbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mvbench", "term": "mvbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_natural2code" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_natural2code", "term": "n2c" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_natural_questions" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_natural_questions", "term": "natural questions" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_natural2code" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_natural2code", "term": "natural2code" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_naturalquestions" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_naturalquestions", "term": "naturalquestions" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_needle_in_a_haystack" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_needle_in_a_haystack", "term": "needle haystack evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_needle_in_a_haystack" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_needle_in_a_haystack", "term": "needle in a haystack" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_next_qa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_next_qa", "term": "next qa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_nexus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_nexus", "term": "nexus" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_needle_in_a_haystack" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_needle_in_a_haystack", "term": "niah" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_nih_multi_needle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_nih_multi_needle", "term": "nih multi needle" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_nl2repo" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_nl2repo", "term": "nl2repo" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_nocaps" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_nocaps", "term": "nocaps" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_nova" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_nova", "term": "nova" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_novel_compiler" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_novel_compiler", "term": "novel compiler" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_natural_questions" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_natural_questions", "term": "nq" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_nucleobench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_nucleobench", "term": "nucleobench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_nuscene" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_nuscene", "term": "nuscene" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_objectron" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_objectron", "term": "objectron" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openbookqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openbookqa", "term": "obqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ocr_bench_v2_chinese" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ocr_bench_v2_chinese", "term": "ocr bench v2 chinese" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ocr_bench_v2_en_zh" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ocr_bench_v2_en_zh", "term": "ocr bench v2 en zh" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ocr_bench_v2_english" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ocr_bench_v2_english", "term": "ocr bench v2 english" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ocrbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ocrbench", "term": "ocrbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ocrbenchv2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ocrbenchv2", "term": "ocrbenchv2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_odinw13" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_odinw13", "term": "odinw13" }, { "kind": "category", "provenance": { "record_ids": [ "category_office_productivity" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_office_productivity", "term": "office productivity" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_officeqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_officeqa", "term": "officeqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_officeqa_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_officeqa_pro", "term": "officeqa pro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ok_vqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ok_vqa", "term": "ok vqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_okapi_mmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_okapi_mmlu", "term": "okapi mmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ok_vqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ok_vqa", "term": "okvqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_olympiadbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_olympiadbench", "term": "olympiadbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_omni_cloze" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_omni_cloze", "term": "omni cloze" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_omnibench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_omnibench", "term": "omnibench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_omnidocbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_omnidocbench", "term": "omnidocbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_omnidocbench_1_5" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_omnidocbench_1_5", "term": "omnidocbench 1 5" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_omnigaia" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_omnigaia", "term": "omnigaia" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_1h_videoqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_1h_videoqa", "term": "one hour videoqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_onemillion_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_onemillion_bench", "term": "onemillion bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_online_mind2web" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_online_mind2web", "term": "online mind2web" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_open_rewrite" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_open_rewrite", "term": "open rewrite" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_chatgpt_factuality" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_chatgpt_factuality", "term": "openai chatgpt factuality" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_code_refactoring_tasks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_code_refactoring_tasks", "term": "openai code refactoring tasks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_code_review_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_code_review_evaluation", "term": "openai code review evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_frontend_preference_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_frontend_preference_evaluation", "term": "openai frontend preference evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_internal_swe_tasks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_internal_swe_tasks", "term": "openai internal swe tasks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_investment_banking_spreadsheet_tasks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_investment_banking_spreadsheet_tasks", "term": "openai investment banking spreadsheet tasks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_management_consulting_tasks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_management_consulting_tasks", "term": "openai management consulting tasks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_mrcr" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_mrcr", "term": "openai mrcr" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_mrcr_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_mrcr_v2", "term": "openai mrcr v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_research_debugging_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_research_debugging_evaluation", "term": "openai research debugging evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_research_engineer_interview" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_research_engineer_interview", "term": "openai research engineer interview" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openbookqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openbookqa", "term": "openbookqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_opencpop" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_opencpop", "term": "opencpop" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openeqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openeqa", "term": "openeqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openrca" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openrca", "term": "openrca" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_opinion_elicitation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_opinion_elicitation", "term": "opinion elicitation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_or_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_or_bench", "term": "or bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_orangesum" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_orangesum", "term": "orangesum" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_oss_fuzz" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_oss_fuzz", "term": "oss fuzz" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_osworld" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_osworld", "term": "osworld" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_osworld_2_0" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_osworld_2_0", "term": "osworld 2 0" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_osworld" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_osworld", "term": "osworld benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_osworld_g" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_osworld_g", "term": "osworld g" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_osworld_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_osworld_verified", "term": "osworld verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_osworld" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_osworld", "term": "osworld verified framework" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_paperbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_paperbench", "term": "paperbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_parametric_cad_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_parametric_cad_bench", "term": "parametric cad bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_passkey_retrieval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_passkey_retrieval", "term": "passkey retrieval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_paws" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_paws", "term": "paws" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_paws_x" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_paws_x", "term": "paws x" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_perception_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_perception_test", "term": "perception test" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_perception_test_mcvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_perception_test_mcvqa", "term": "perception test mcvqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_perceptionbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_perceptionbench", "term": "perceptionbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_perception_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_perception_test", "term": "perceptiontest" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_personqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_personqa", "term": "personqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_phonology_bench_g2p" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_phonology_bench_g2p", "term": "phonology bench g2p" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_phrase_and_password_protection" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_phrase_and_password_protection", "term": "phrase and password protection" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_physicsfinals" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_physicsfinals", "term": "physicsfinals" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_phyx" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_phyx", "term": "phyx" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_pile_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_pile_test", "term": "pile test" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_piqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_piqa", "term": "piqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plawbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plawbench", "term": "plawbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plm_videobench_fgqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plm_videobench_fgqa", "term": "plm videobench fgqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plm_videobench_rcap" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plm_videobench_rcap", "term": "plm videobench rcap" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plm_videobench_rdcap" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plm_videobench_rdcap", "term": "plm videobench rdcap" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plm_videobench_rtloc" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plm_videobench_rtloc", "term": "plm videobench rtloc" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plm_videobench_sgqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plm_videobench_sgqa", "term": "plm videobench sgqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_pmc_vqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_pmc_vqa", "term": "pmc vqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_pn_summary" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_pn_summary", "term": "pn summary" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_poly_guard_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_poly_guard_bench", "term": "poly guard bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_polymath" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_polymath", "term": "polymath" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_pope" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_pope", "term": "pope" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_prbench_finance" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_prbench_finance", "term": "prbench finance" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_prbench_legal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_prbench_legal", "term": "prbench legal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_presentbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_presentbench", "term": "presentbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_pretraining_documents" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_pretraining_documents", "term": "pretraining documents" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_production_jailbreaks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_production_jailbreaks", "term": "production jailbreaks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_professional_ctf" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_professional_ctf", "term": "professional ctf" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_professional_ctfs" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_professional_ctfs", "term": "professional ctfs" }, { "kind": "category", "provenance": { "record_ids": [ "category_professional_tasks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_professional_tasks", "term": "professional tasks" }, { "kind": "category", "provenance": { "record_ids": [ "category_professional_work" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_professional_work", "term": "professional work" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_programbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_programbench", "term": "programbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_project_gutenberg" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_project_gutenberg", "term": "project gutenberg" }, { "kind": "category", "provenance": { "record_ids": [ "category_prompt_injection" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_prompt_injection", "term": "prompt injection" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_prompted_evaluation_awareness" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_prompted_evaluation_awareness", "term": "prompted evaluation awareness" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_proteingym_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_proteingym_hard", "term": "proteingym hard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_labbench_protocolqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_labbench_protocolqa", "term": "protocol understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lab_bench_protocolqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lab_bench_protocolqa", "term": "protocolqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_protocolqa_open_ended" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_protocolqa_open_ended", "term": "protocolqa open ended" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cybench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cybench", "term": "public cyber capabilities benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_pubmedqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_pubmedqa", "term": "pubmedqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qasper" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qasper", "term": "qasper" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_quac" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_quac", "term": "quac" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_quadruped_locomotion" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_quadruped_locomotion", "term": "quadruped locomotion" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_quadruped_reinforcement_learning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_quadruped_reinforcement_learning", "term": "quadruped reinforcement learning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qualcomminteractive" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qualcomminteractive", "term": "qualcomminteractive" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_quality" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_quality", "term": "quality" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_quality" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_quality", "term": "quality question answering with long input texts yes" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_quantbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_quantbench", "term": "quantbench" }, { "kind": "category", "provenance": { "record_ids": [ "category_question_answering" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_question_answering", "term": "question answering" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_chartqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_chartqa", "term": "question answering about charts and graphs" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ai2d" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ai2d", "term": "question answering about science diagrams" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenblenderbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenblenderbench", "term": "qwenblenderbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenchinesebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenchinesebench", "term": "qwenchinesebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenclaw" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenclaw", "term": "qwenclaw" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenclawbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenclawbench", "term": "qwenclawbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenqoderbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenqoderbench", "term": "qwenqoderbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenreactbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenreactbench", "term": "qwenreactbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwensvg" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwensvg", "term": "qwensvg" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwensvgbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwensvgbench", "term": "qwensvgbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenswebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenswebench", "term": "qwenswebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenvision2code" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenvision2code", "term": "qwenvision2code" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenvisualoffice" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenvisualoffice", "term": "qwenvisualoffice" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenwebbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenwebbench", "term": "qwenwebbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenwebdev" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenwebdev", "term": "qwenwebdev" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_qwenworldbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_qwenworldbench", "term": "qwenworldbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_race" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_race", "term": "race" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_race_high" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_race_high", "term": "race h" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_race_high" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_race_high", "term": "race high" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_race_middle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_race_middle", "term": "race middle" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_rakuten_swe_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_rakuten_swe_bench", "term": "rakuten swe bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_random_chat_questions" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_random_chat_questions", "term": "random chat questions" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plm_videobench_rcap" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plm_videobench_rcap", "term": "rcap" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plm_videobench_rdcap" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plm_videobench_rdcap", "term": "rdcap" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_re_bench_subset" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_re_bench_subset", "term": "re bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_re_bench_subset" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_re_bench_subset", "term": "re bench subset" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_race_high" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_race_high", "term": "reading comprehension" }, { "kind": "category", "provenance": { "record_ids": [ "category_reading_comprehension" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_reading_comprehension", "term": "reading comprehension" }, { "kind": "category", "provenance": { "record_ids": [ "category_reading_comprehension_and_discrete_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_reading_comprehension_and_discrete_reasoning", "term": "reading comprehension and discrete reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_drop" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_drop", "term": "reading comprehension arithmetic" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_real_world_finance" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_real_world_finance", "term": "real world finance" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_real_world_finance_v1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_real_world_finance_v1", "term": "real world finance v1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_real_world_finance_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_real_world_finance_v2", "term": "real world finance v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_realtoxicity" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_realtoxicity", "term": "realtoxicity" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_realworldqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_realworldqa", "term": "realworldqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmlu", "term": "reasoning" }, { "kind": "category", "provenance": { "record_ids": [ "category_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_reasoning", "term": "reasoning" }, { "kind": "category", "provenance": { "record_ids": [ "category_reasoning_and_knowledge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_reasoning_and_knowledge", "term": "reasoning and knowledge" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_recreationbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_recreationbench", "term": "recreationbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_redcode" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_redcode", "term": "redcode" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_refadv_s" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_refadv_s", "term": "refadv s" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_refcoco" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_refcoco", "term": "refcoco" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_refcoco_plus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_refcoco_plus", "term": "refcoco plus" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_refcocog" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_refcocog", "term": "refcocog" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_refspatialbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_refspatialbench", "term": "refspatialbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_repoeval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_repoeval", "term": "repoeval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_graphwalks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_graphwalks", "term": "reported with bfs and parent node variants" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_graphwalks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_graphwalks", "term": "reported with bfs and parents task variants" }, { "kind": "category", "provenance": { "record_ids": [ "category_research_engineering" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_research_engineering", "term": "research engineering" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_research_sabotage" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_research_sabotage", "term": "research sabotage" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_research_sandbagging" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_research_sandbagging", "term": "research sandbagging" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_research_sandbagging" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_research_sandbagging", "term": "research sandbagging evaluation" }, { "kind": "category", "provenance": { "record_ids": [ "category_safety_and_responsibility" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_safety_and_responsibility", "term": "responsible ai" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_reward_hack_prone_coding_tasks_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_reward_hack_prone_coding_tasks_v2", "term": "reward hack prone coding tasks v2" }, { "kind": "category", "provenance": { "record_ids": [ "category_reward_hacking" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_reward_hacking", "term": "reward hacking" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_riemannbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_riemannbench", "term": "riemannbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_robospatialhome" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_robospatialhome", "term": "robospatialhome" }, { "kind": "category", "provenance": { "record_ids": [ "category_robustness_and_safety" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_robustness_and_safety", "term": "robustness and safety" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_rocstories" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_rocstories", "term": "roc stories" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_rocstories" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_rocstories", "term": "rocstories" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plm_videobench_rtloc" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plm_videobench_rtloc", "term": "rtloc" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_rul_muchomusic" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_rul_muchomusic", "term": "rul muchomusic" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ruler" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ruler", "term": "ruler" }, { "kind": "category", "provenance": { "record_ids": [ "category_sabotage_capability" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_sabotage_capability", "term": "sabotage capability" }, { "kind": "category", "provenance": { "record_ids": [ "category_safeguard_evasion" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_safeguard_evasion", "term": "safeguard evasion" }, { "kind": "category", "provenance": { "record_ids": [ "category_safety_and_alignment" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_safety_and_alignment", "term": "safety and alignment" }, { "kind": "category", "provenance": { "record_ids": [ "category_safety_and_responsibility" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_safety_and_responsibility", "term": "safety and responsibility" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_safim" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_safim", "term": "safim" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_samsum" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_samsum", "term": "samsum" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_sat_math" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_sat_math", "term": "sat math" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_sat_reading" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_sat_reading", "term": "sat reading" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_save_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_save_bench", "term": "save bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcp_atlas" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcp_atlas", "term": "scale mcp atlas" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_scicode" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_scicode", "term": "scicode" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ai2d" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ai2d", "term": "science diagram benchmark" }, { "kind": "category", "provenance": { "record_ids": [ "category_science_diagram_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_science_diagram_reasoning", "term": "science diagram reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_ai2d" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_ai2d", "term": "science diagrams" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_screenspot" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_screenspot", "term": "screenspot" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_screenspot_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_screenspot_pro", "term": "screenspot pro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_seal_0" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_seal_0", "term": "seal 0" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_sec_bench_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_sec_bench_pro", "term": "sec bench pro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_seccodebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_seccodebench", "term": "seccodebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_secure_bio_lab_assistance_benchmark_multimodal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_secure_bio_lab_assistance_benchmark_multimodal", "term": "secure bio lab assistance benchmark multimodal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_secure_code_generation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_secure_code_generation", "term": "secure code generation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_securebio_dna_synthesis_screening_evasion" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_securebio_dna_synthesis_screening_evasion", "term": "securebio dna synthesis screening evasion" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_securebio_virology_capability_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_securebio_virology_capability_test", "term": "securebio virology capability test" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_securebio_virology_capability_test" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_securebio_virology_capability_test", "term": "securebio virology capability test vct" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_seed_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_seed_bench", "term": "seed" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_seed_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_seed_bench", "term": "seed bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_seed_tts_eval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_seed_tts_eval", "term": "seed tts eval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_lab_bench_seqqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_lab_bench_seqqa", "term": "seqqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_severe_cybermisuse_refusal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_severe_cybermisuse_refusal", "term": "severe cybermisuse refusal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_sfiction" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_sfiction", "term": "sfiction" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_plm_videobench_sgqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_plm_videobench_sgqa", "term": "sgqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_shade_arena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_shade_arena", "term": "shade arena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_short_horizon_computational_biology_tasks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_short_horizon_computational_biology_tasks", "term": "short horizon computational biology tasks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_sifo" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_sifo", "term": "sifo" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_sifo_multiturn" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_sifo_multiturn", "term": "sifo multiturn" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_simpleqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_simpleqa", "term": "simple qa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_simpleqa_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_simpleqa_verified", "term": "simple qa verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_simpleqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_simpleqa", "term": "simpleqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_simpleqa_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_simpleqa_verified", "term": "simpleqa verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_simplevqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_simplevqa", "term": "simplevqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_browsecomp" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_browsecomp", "term": "single agent" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_skillsbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_skillsbench", "term": "skillsbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_slake" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_slake", "term": "slake" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_social_engineering_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_social_engineering_evaluation", "term": "social engineering evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_social_engineering_refusal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_social_engineering_refusal", "term": "social engineering refusal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_social_iqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_social_iqa", "term": "social iqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_socialiqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_socialiqa", "term": "socialiqa" }, { "kind": "category", "provenance": { "record_ids": [ "category_software_engineering" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_software_engineering", "term": "software engineering" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_verified", "term": "software engineering bench verified" }, { "kind": "category", "provenance": { "record_ids": [ "category_software_reliability" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_software_reliability", "term": "software reliability" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_songformbench_chinese" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_songformbench_chinese", "term": "songformbench chinese" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_songformbench_harmonixset" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_songformbench_harmonixset", "term": "songformbench harmonixset" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_latchbio_bioinformatics" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_latchbio_bioinformatics", "term": "spatialbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_spatialsense_vqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_spatialsense_vqa", "term": "spatialsense vqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_speechrole" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_speechrole", "term": "speechrole" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_spider" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_spider", "term": "spider" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_spreadsheetbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_spreadsheetbench", "term": "spreadsheetbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_spreadsheetbench_v1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_spreadsheetbench_v1", "term": "spreadsheetbench v1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_squad_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_squad_v2", "term": "squad v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_squality" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_squality", "term": "squality" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_standard_refusal_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_standard_refusal_evaluation", "term": "standard refusal evaluation" }, { "kind": "category", "provenance": { "record_ids": [ "category_standardized_analytical_writing" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_standardized_analytical_writing", "term": "standardized analytical writing" }, { "kind": "category", "provenance": { "record_ids": [ "category_standardized_legal_knowledge" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_standardized_legal_knowledge", "term": "standardized legal knowledge" }, { "kind": "category", "provenance": { "record_ids": [ "category_standardized_legal_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_standardized_legal_reasoning", "term": "standardized legal reasoning" }, { "kind": "category", "provenance": { "record_ids": [ "category_standardized_quantitative_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_standardized_quantitative_reasoning", "term": "standardized quantitative reasoning" }, { "kind": "category", "provenance": { "record_ids": [ "category_standardized_verbal_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_standardized_verbal_reasoning", "term": "standardized verbal reasoning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_star_video_qa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_star_video_qa", "term": "star" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_star_video_qa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_star_video_qa", "term": "star video qa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_strongreject" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_strongreject", "term": "strongreject" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_strongreject_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_strongreject_v2", "term": "strongreject v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_subversion_strategy" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_subversion_strategy", "term": "subversion strategy" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_subversion_strategy" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_subversion_strategy", "term": "subversion strategy task" }, { "kind": "category", "provenance": { "record_ids": [ "category_summarization" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_summarization", "term": "summarization" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_sunrgbd" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_sunrgbd", "term": "sunrgbd" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_super_agent_benchmark" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_super_agent_benchmark", "term": "super agent benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_superglue" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_superglue", "term": "superglue" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_supergpqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_supergpqa", "term": "supergpqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_surds" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_surds", "term": "surds" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_agentless" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_agentless", "term": "swe agentless" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_verified", "term": "swe bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_verified_hard_subset" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_verified_hard_subset", "term": "swe bench benchmark" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_verified", "term": "swe bench is expanded as software engineering bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_multilingual", "term": "swe bench multilingual" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_multimodal" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_multimodal", "term": "swe bench multimodal" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_pro", "term": "swe bench pro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_pro", "term": "swe bench pro public" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_verified", "term": "swe bench verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_verified_hard" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_verified_hard", "term": "swe bench verified hard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_verified_hard_subset" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_verified_hard_subset", "term": "swe bench verified hard subset" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_lancer" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_lancer", "term": "swe lancer" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_lancer_diamond" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_lancer_diamond", "term": "swe lancer diamond" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_lancer_ic_diamond" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_lancer_ic_diamond", "term": "swe lancer ic diamond" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_lancer_ic_swe" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_lancer_ic_swe", "term": "swe lancer ic swe" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_multilingual", "term": "swe multilingual" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_pi_agent" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_pi_agent", "term": "swe pi agent" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_pi_synthetic" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_pi_synthetic", "term": "swe pi synthetic" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_pro", "term": "swe pro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_verified", "term": "swe verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swebench_pretrain" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swebench_pretrain", "term": "swebench pretrain" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_verified", "term": "swebench verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_sycophancy_eval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_sycophancy_eval", "term": "sycophancy eval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_synthesis_screening_evasion" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_synthesis_screening_evasion", "term": "synthesis screening evasion" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_securebio_dna_synthesis_screening_evasion" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_securebio_dna_synthesis_screening_evasion", "term": "synthesis screening evasion evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_synthesis_screening_evasion" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_synthesis_screening_evasion", "term": "synthesis screening evasion task" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_synthetic_honeypots" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_synthetic_honeypots", "term": "synthetic honeypots" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_real_world_finance" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_real_world_finance", "term": "table 2 12 1 a identifies it as an internal evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_real_world_finance" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_real_world_finance", "term": "table 2 14 2 identifies it as an internal evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tallyqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tallyqa", "term": "tallyqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tat_dqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tat_dqa", "term": "tat dqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bench", "term": "tau bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tau_bench_airline" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tau_bench_airline", "term": "tau bench airline" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bench", "term": "tau bench bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tau_bench_retail" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tau_bench_retail", "term": "tau bench retail" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tau_bench_telecom" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tau_bench_telecom", "term": "tau bench telecom" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_bench", "term": "tau retail" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tau2_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tau2_bench", "term": "tau2 bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench_retail" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench_retail", "term": "tau2 bench retail" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench_telecom" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench_telecom", "term": "tau2 bench telecom" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tau3_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tau3_bench", "term": "tau3 bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tau_bench_airline" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tau_bench_airline", "term": "taubench airline" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tau_bench_retail" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tau_bench_retail", "term": "taubench retail" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tempcompass" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tempcompass", "term": "tempcompass" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_temporalbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_temporalbench", "term": "temporalbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_terminal_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_terminal_bench", "term": "terminal bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_terminal_bench_2_0" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_terminal_bench_2_0", "term": "terminal bench 2 0" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_terminal_bench_2_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_terminal_bench_2_1", "term": "terminal bench 2 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_terminal_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_terminal_bench", "term": "terminalbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mathvista" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mathvista", "term": "testmini" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_text_arena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_text_arena", "term": "text arena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_text_based_reinforcement_learning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_text_based_reinforcement_learning", "term": "text based reinforcement learning" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_text_based_reinforcement_learning_task" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_text_based_reinforcement_learning_task", "term": "text based reinforcement learning task" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_text_based_reinforcement_learning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_text_based_reinforcement_learning", "term": "text based rl" }, { "kind": "category", "provenance": { "record_ids": [ "category_text_quality" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_text_quality", "term": "text quality" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_textvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_textvqa", "term": "text vqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_textvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_textvqa", "term": "textvqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_thaisum" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_thaisum", "term": "thaisum" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_agi_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_agi_2", "term": "the launch footnote prints arc agi 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcp_atlas" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcp_atlas", "term": "the launch footnote prints mcp atlas" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vending_bench_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vending_bench_2", "term": "the launch footnote prints vending bench 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_browsecomp_plus" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_browsecomp_plus", "term": "the launch methodology calls one configuration a fetch enabled version of browsecomp plus" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle", "term": "the launch page abbreviates it as hle in an update note" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vending_bench_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vending_bench_2", "term": "the launch page also prints vending bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vending_bench_arena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vending_bench_arena", "term": "the launch page distinguishes this competitive arena evaluation from vending bench 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_openai_mrcr_v2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_openai_mrcr_v2", "term": "the launch page prints mrcr v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_officeqa_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_officeqa_pro", "term": "the launch page prints officeqa pro in a databricks quotation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_swe_bench_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_swe_bench_multilingual", "term": "the launch page prints swe bench multilingual" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_record_2_bench_airline_original" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_record_2_bench_airline_original", "term": "the launch page refers to the airline scenario as 2 bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_osworld_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_osworld_verified", "term": "the launch section initially discusses the osworld benchmark family and states that this model uses osworld verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vending_bench_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vending_bench_2", "term": "the section body identifies the benchmark as vending bench 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mcp_atlas" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mcp_atlas", "term": "the section body prints mcp atlas" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_medcalc_bench_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_medcalc_bench_verified", "term": "the section first prints medcalc bench and identifies use of the latest verified dataset" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_terminal_bench_2_0" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_terminal_bench_2_0", "term": "the section heading prints terminal bench while the method names terminal bench 2 0" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_hle" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_hle", "term": "the summary table abbreviates the benchmark as hle" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_agi_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_agi_1", "term": "the summary table groups the family under arc agi" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mmmu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mmmu", "term": "the summary table identifies the validation split" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_arc_agi_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_arc_agi_2", "term": "the summary table prints arc agi 2 verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_cybergym" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_cybergym", "term": "the text also calls the task collection the cybergym suite" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_theoremqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_theoremqa", "term": "theoremqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_time_series_forecasting" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_time_series_forecasting", "term": "time series forecasting" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tir_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tir_bench", "term": "tir bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tldr9" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tldr9", "term": "tldr9" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_toefl" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_toefl", "term": "toefl" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tomato" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tomato", "term": "tomato" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tool_decathlon" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tool_decathlon", "term": "tool decathlon" }, { "kind": "category", "provenance": { "record_ids": [ "category_tool_use" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_tool_use", "term": "tool use" }, { "kind": "category", "provenance": { "record_ids": [ "category_tool_use_and_agents" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_tool_use_and_agents", "term": "tool use and agents" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tool_use_prompt_injection_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tool_use_prompt_injection_evaluation", "term": "tool use evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tool_use_prompt_injection_evaluation" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tool_use_prompt_injection_evaluation", "term": "tool use prompt injection evaluation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_toolathlon" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_toolathlon", "term": "toolathlon" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_toolathlon_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_toolathlon_verified", "term": "toolathlon verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_toolcallformatifbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_toolcallformatifbench", "term": "toolcallformatifbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_toxigen" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_toxigen", "term": "toxigen" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_triviaqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_triviaqa", "term": "tqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_mgsm" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_mgsm", "term": "translated version of the math benchmark gsm8k" }, { "kind": "category", "provenance": { "record_ids": [ "category_multilingual" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_multilingual", "term": "translation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_triviaqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_triviaqa", "term": "triviaqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_triviaqa_wiki" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_triviaqa_wiki", "term": "triviaqa wiki" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_truthfulqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_truthfulqa", "term": "truthfulqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_turkishmmlu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_turkishmmlu", "term": "turkishmmlu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tutor_jailbreaks" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tutor_jailbreaks", "term": "tutor jailbreaks" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tvbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tvbench", "term": "tvbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_tydiqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_tydiqa", "term": "tydiqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_unisvg" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_unisvg", "term": "unisvg" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_united_states_medical_licensing_examination" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_united_states_medical_licensing_examination", "term": "united states medical licensing examination" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_uro_bench_pro" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_uro_bench_pro", "term": "uro bench pro" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_usamo_2026" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_usamo_2026", "term": "usa mathematical olympiad 2026" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_usamo_2025" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_usamo_2025", "term": "usamo 2025" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_usamo_2026" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_usamo_2026", "term": "usamo 2026" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_usamo_2026" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_usamo_2026", "term": "usamo is expanded as usa mathematical olympiad" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gdpval_aa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gdpval_aa", "term": "uses tasks from the gdpval gold database" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_gdpval_aa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_gdpval_aa", "term": "uses tasks from the gdpval gold dataset" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_united_states_medical_licensing_examination" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_united_states_medical_licensing_examination", "term": "usmle" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_v" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_v", "term": "v" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_v_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_v_bench", "term": "v bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "term": "validation" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vals_ai_finance_agent" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vals_ai_finance_agent", "term": "vals ai finance agent" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vals_ai_finance_agent" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vals_ai_finance_agent", "term": "vals ai public leaderboard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vatex" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vatex", "term": "vatex" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vatex_zh" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vatex_zh", "term": "vatex zh" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vct" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vct", "term": "vct" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vending_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vending_bench", "term": "vending bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vending_bench_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vending_bench_2", "term": "vending bench 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vending_bench_arena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vending_bench_arena", "term": "vending bench arena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vending_bench_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vending_bench_2", "term": "vendingbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vibecodebench_v1_1" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vibecodebench_v1_1", "term": "vibecodebench v1 1" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vibench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vibench", "term": "vibench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_video_mme" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_video_mme", "term": "video mme" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_video_mmmu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_video_mmmu", "term": "video mmmu" }, { "kind": "category", "provenance": { "record_ids": [ "category_video_understanding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_video_understanding", "term": "video understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_videodr_w_search" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_videodr_w_search", "term": "videodr w search" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_videohallucer" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_videohallucer", "term": "videohallucer" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_videomme" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_videomme", "term": "videomme" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_videomme_v2_w_sub" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_videomme_v2_w_sub", "term": "videomme v2 w sub" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_videomme_with_audio" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_videomme_with_audio", "term": "videomme with audio" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_videomme_with_subtitles" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_videomme_with_subtitles", "term": "videomme with subtitles" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_videomme_without_subtitles" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_videomme_without_subtitles", "term": "videomme without subtitles" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_videommmu" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_videommmu", "term": "videommmu" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_visfactor" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_visfactor", "term": "visfactor" }, { "kind": "category", "provenance": { "record_ids": [ "category_multimodal_understanding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_multimodal_understanding", "term": "vision language" }, { "kind": "category", "provenance": { "record_ids": [ "category_vision_language_understanding" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "category_vision_language_understanding", "term": "vision language understanding" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vision2web" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vision2web", "term": "vision2web" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_massive_multi_discipline_multimodal_understanding_and_reasoning", "term": "visual question answering" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_visulogic" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_visulogic", "term": "visulogic" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vita_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vita_bench", "term": "vita bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vizwiz" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vizwiz", "term": "vizwiz" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vladbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vladbench", "term": "vladbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vlmsarebiased" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vlmsarebiased", "term": "vlmsarebiased" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vlmsareblind" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vlmsareblind", "term": "vlmsareblind" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vmqa_vct" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vmqa_vct", "term": "vmqa vct" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_voicebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_voicebench", "term": "voicebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_voicebench_commoneval" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_voicebench_commoneval", "term": "voicebench commoneval" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vqav2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vqav2", "term": "vqa v2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vqav2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vqav2", "term": "vqav2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vsi_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vsi_bench", "term": "vsi bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_vsr" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_vsr", "term": "vsr" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_we_math" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_we_math", "term": "we math" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_webarena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_webarena", "term": "webarena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_webarena_verified" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_webarena_verified", "term": "webarena verified" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_webdev_arena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_webdev_arena", "term": "webdev arena" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_webdev_arena" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_webdev_arena", "term": "webdev arena leaderboard" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_webvoyager" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_webvoyager", "term": "webvoyager" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wenetspeech" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wenetspeech", "term": "wenetspeech" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_widesearch" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_widesearch", "term": "widesearch" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wikipedia_english" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wikipedia_english", "term": "wikipedia en" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wikipedia_english" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wikipedia_english", "term": "wikipedia english" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wikisummary" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wikisummary", "term": "wikisummary" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wildchat" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wildchat", "term": "wildchat" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wildchat" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wildchat", "term": "wildchat non toxic" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wildchat" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wildchat", "term": "wildchat toxic" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wildspeech_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wildspeech_bench", "term": "wildspeech bench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_winobias_type_1_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_winobias_type_1_2", "term": "winobias 1 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_winobias_type_2_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_winobias_type_2_2", "term": "winobias 2 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_winobias_type_1_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_winobias_type_1_2", "term": "winobias type 1 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_winobias_type_2_2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_winobias_type_2_2", "term": "winobias type 2 2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_winogender" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_winogender", "term": "winogender" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_winogrande" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_winogrande", "term": "winogrande" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wmdp_bio" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wmdp_bio", "term": "wmdp bio" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wmdp_biology" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wmdp_biology", "term": "wmdp biology" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wmdp_chem" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wmdp_chem", "term": "wmdp chem" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wmdp_chemistry" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wmdp_chemistry", "term": "wmdp chemistry" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wmdp_cyber" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wmdp_cyber", "term": "wmdp cyber" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wmt23" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wmt23", "term": "wmt23" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_wmt24" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_wmt24", "term": "wmt24" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_workspacebench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_workspacebench", "term": "workspacebench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_worldsense" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_worldsense", "term": "worldsense" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_worldvqa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_worldvqa", "term": "worldvqa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_writingbench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_writingbench", "term": "writingbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_xcopa" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_xcopa", "term": "xcopa" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_xquad" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_xquad", "term": "xquad" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_xstest" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_xstest", "term": "xstest" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_xstest" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_xstest", "term": "xstest evaluations" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_xstorycloze" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_xstorycloze", "term": "xstorycloze" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_xsum" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_xsum", "term": "xsum" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_xwinograd" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_xwinograd", "term": "xwinograd" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_youcook2" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_youcook2", "term": "youcook2" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_automation_bench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_automation_bench", "term": "zapier automationbench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_zebralogic" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_zebralogic", "term": "zebralogic" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_zerobench" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_zerobench", "term": "zerobench" }, { "kind": "benchmark", "provenance": { "record_ids": [ "benchmark_zerobench_sub" ], "record_type": "normalized_search_term", "source_file": "data/catalog.yaml" }, "record_id": "benchmark_zerobench_sub", "term": "zerobench sub" } ], "sources": [ { "id": "source_anthropic_launch_anthropic_20240304_introducing_the_next_generation_of_claude", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20240304_introducing_the_next_generation_of_claude", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "source_anthropic_launch_anthropic_20240313_claude_3_haiku_our_fastest_model_yet", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20240313_claude_3_haiku_our_fastest_model_yet" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-13", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20240313_claude_3_haiku_our_fastest_model_yet", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-3-haiku" }, { "id": "source_anthropic_launch_anthropic_20240621_claude_3_5_sonnet", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20240621_claude_3_5_sonnet", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-3-5-sonnet" }, { "id": "source_anthropic_launch_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/3-5-models-and-computer-use" }, { "id": "source_anthropic_launch_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-3-7-sonnet" }, { "id": "source_anthropic_launch_anthropic_20250522_introducing_claude_4", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20250522_introducing_claude_4" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20250522_introducing_claude_4", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-4" }, { "id": "source_anthropic_launch_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-06", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-gov-models-for-u-s-national-security-customers" }, { "id": "source_anthropic_launch_anthropic_20250805_claude_opus_4_1", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20250805_claude_opus_4_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20250805_claude_opus_4_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-4-1" }, { "id": "source_anthropic_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20250929_introducing_claude_sonnet_4_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "source_anthropic_launch_anthropic_20251015_introducing_claude_haiku_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20251015_introducing_claude_haiku_4_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-haiku-4-5" }, { "id": "source_anthropic_launch_anthropic_20251124_introducing_claude_opus_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20251124_introducing_claude_opus_4_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-4-5" }, { "id": "source_anthropic_launch_anthropic_20260205_introducing_claude_opus_4_6", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20260205_introducing_claude_opus_4_6", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-4-6" }, { "id": "source_anthropic_launch_anthropic_20260217_introducing_claude_sonnet_4_6", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20260217_introducing_claude_sonnet_4_6", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-sonnet-4-6" }, { "id": "source_anthropic_launch_anthropic_20260416_introducing_claude_opus_4_7", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20260416_introducing_claude_opus_4_7", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-4-7" }, { "id": "source_anthropic_launch_anthropic_20260528_introducing_claude_opus_4_8", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20260528_introducing_claude_opus_4_8", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-4-8" }, { "id": "source_anthropic_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-fable-5-mythos-5" }, { "id": "source_anthropic_launch_anthropic_20260630_introducing_claude_sonnet_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20260630_introducing_claude_sonnet_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-sonnet-5" }, { "id": "source_anthropic_launch_anthropic_20260630_redeploying_fable_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20260630_redeploying_fable_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20260630_redeploying_fable_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/redeploying-fable-5" }, { "id": "source_anthropic_launch_anthropic_20260724_introducing_claude_opus_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_launch_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "revision": { "id": "revision_anthropic_revision_launch_anthropic_20260724_introducing_claude_opus_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-5" }, { "id": "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "revision": { "id": "revision_anthropic_revision_technical_0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b", "identifier": "sha256:0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-opus-5-system-card" }, { "id": "source_anthropic_technical_0ce49aa473c3037455b45579980fd0b55708c1066df3f5f99f92bb3169d9b9b7", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_0ce49aa473c3037455b45579980fd0b55708c1066df3f5f99f92bb3169d9b9b7" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "revision": { "id": "revision_anthropic_revision_technical_0ce49aa473c3037455b45579980fd0b55708c1066df3f5f99f92bb3169d9b9b7", "identifier": "sha256:0ce49aa473c3037455b45579980fd0b55708c1066df3f5f99f92bb3169d9b9b7" }, "source_type": "system_card", "url": "https://www-cdn.anthropic.com/9e6a1044980d8c4ed85669faf9c2a8342e2e9f1e/Claude-Sonnet-5-System-Card.pdf" }, { "id": "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "revision": { "id": "revision_anthropic_revision_technical_0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f", "identifier": "sha256:0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-opus-4-8-system-card" }, { "id": "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "revision": { "id": "revision_anthropic_revision_technical_1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141", "identifier": "sha256:1b9ae5f4f66a65c1b37a031179af3dba35959471a3a5d195e4fe700b87558141" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-haiku-4-5-system-card" }, { "id": "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "revision": { "id": "revision_anthropic_revision_technical_33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed", "identifier": "sha256:33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-sonnet-5-system-card" }, { "id": "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "revision": { "id": "revision_anthropic_revision_technical_33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8", "identifier": "sha256:33be92efcb58f901d9d2c17e61bf277afbd514a1ac70a642f21e918c21ca25a8" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-opus-4-5-system-card" }, { "id": "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "revision": { "id": "revision_anthropic_revision_technical_4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c", "identifier": "sha256:4db67c6b6aea0a87e7f8a784b83fc05a0f1a61a3e87615fe7956e3486b951b3c" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-opus-4-6-system-card" }, { "id": "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "revision": { "id": "revision_anthropic_revision_technical_5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c", "identifier": "sha256:5425dde49b649916000e38fb86c4721cfe31a8bdfe30bbb98e875fa3d654804c" }, "source_type": "model_card", "url": "https://assets.anthropic.com/m/1cd9d098ac3e6467/original/Claude-3-Model-Card-October-Addendum.pdf" }, { "id": "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "revision": { "id": "revision_anthropic_revision_technical_672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6", "identifier": "sha256:672339ab327e8798578b3559e8489561db5ad25e81b427de4e306393108e90b6" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-sonnet-4-6-system-card" }, { "id": "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "revision": { "id": "revision_anthropic_revision_technical_82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6", "identifier": "sha256:82947a46fc2d1b9a1d69501ef973fc3c5b328b703b9a17bd8a143c2672f8a1a6" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-3-7-sonnet-system-card" }, { "id": "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "revision": { "id": "revision_anthropic_revision_technical_88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d", "identifier": "sha256:88ae851f53c3034d436816164ce49a876b333fa9b550fb7ef29838182b59ba7d" }, "source_type": "model_card", "url": "https://www.anthropic.com/claude-3-model-card" }, { "id": "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "revision": { "id": "revision_anthropic_revision_technical_a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2", "identifier": "sha256:a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-opus-4-7-system-card" }, { "id": "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "revision": { "id": "revision_anthropic_revision_technical_ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03", "identifier": "sha256:ac396c7f155dcbfb33ceb342b48647e330ca518c01c7f508001d6aad93257a03" }, "source_type": "model_card", "url": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf" }, { "id": "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "revision": { "id": "revision_anthropic_revision_technical_b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2", "identifier": "sha256:b2b5beba3ed386a58de760e664f5c98b90a6ce1275c7ee66da8beb70e106e0f2" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-opus-4-1-system-card" }, { "id": "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "revision": { "id": "revision_anthropic_revision_technical_d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e", "identifier": "sha256:d5d1ce28fea101ae3b201a5d4da100090a5d6b73246f7da820cafd7dbb11100e" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-sonnet-4-5-system-card" }, { "id": "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_anthropic_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "revision": { "id": "revision_anthropic_revision_technical_f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca", "identifier": "sha256:f95d413845ad8624f384ba026963f2bad2158f10f2626575bb45e823e3c2e0ca" }, "source_type": "system_card", "url": "https://www.anthropic.com/claude-fable-5-mythos-5-system-card" }, { "id": "source_candidate_anthropic_20240304_introducing_the_next_generation_of_claude", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20240304_introducing_the_next_generation_of_claude" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-04", "revision": { "id": "revision_candidate_anthropic_20240304_introducing_the_next_generation_of_claude", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "source_candidate_anthropic_20240313_claude_3_haiku_our_fastest_model_yet", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20240313_claude_3_haiku_our_fastest_model_yet" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-13", "revision": { "id": "revision_candidate_anthropic_20240313_claude_3_haiku_our_fastest_model_yet", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-3-haiku" }, { "id": "source_candidate_anthropic_20240621_claude_3_5_sonnet", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20240621_claude_3_5_sonnet" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-21", "revision": { "id": "revision_candidate_anthropic_20240621_claude_3_5_sonnet", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-3-5-sonnet" }, { "id": "source_candidate_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-22", "revision": { "id": "revision_candidate_anthropic_20241022_introducing_computer_use_a_new_claude_3_5_sonnet_and", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/3-5-models-and-computer-use" }, { "id": "source_candidate_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20250224_claude_3_7_sonnet_and_claude_code" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "revision": { "id": "revision_candidate_anthropic_20250224_claude_3_7_sonnet_and_claude_code", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-3-7-sonnet" }, { "id": "source_candidate_anthropic_20250522_introducing_claude_4", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20250522_introducing_claude_4" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-22", "revision": { "id": "revision_candidate_anthropic_20250522_introducing_claude_4", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-4" }, { "id": "source_candidate_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-06", "revision": { "id": "revision_candidate_anthropic_20250606_claude_gov_models_for_u_s_national_security_customer", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-gov-models-for-u-s-national-security-customers" }, { "id": "source_candidate_anthropic_20250805_claude_opus_4_1", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20250805_claude_opus_4_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "revision": { "id": "revision_candidate_anthropic_20250805_claude_opus_4_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-4-1" }, { "id": "source_candidate_anthropic_20250929_introducing_claude_sonnet_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20250929_introducing_claude_sonnet_4_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "revision": { "id": "revision_candidate_anthropic_20250929_introducing_claude_sonnet_4_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "source_candidate_anthropic_20251015_introducing_claude_haiku_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20251015_introducing_claude_haiku_4_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "revision": { "id": "revision_candidate_anthropic_20251015_introducing_claude_haiku_4_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-haiku-4-5" }, { "id": "source_candidate_anthropic_20251124_introducing_claude_opus_4_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20251124_introducing_claude_opus_4_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-24", "revision": { "id": "revision_candidate_anthropic_20251124_introducing_claude_opus_4_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-4-5" }, { "id": "source_candidate_anthropic_20260205_introducing_claude_opus_4_6", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20260205_introducing_claude_opus_4_6" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "revision": { "id": "revision_candidate_anthropic_20260205_introducing_claude_opus_4_6", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-4-6" }, { "id": "source_candidate_anthropic_20260217_introducing_claude_sonnet_4_6", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20260217_introducing_claude_sonnet_4_6" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-17", "revision": { "id": "revision_candidate_anthropic_20260217_introducing_claude_sonnet_4_6", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-sonnet-4-6" }, { "id": "source_candidate_anthropic_20260416_introducing_claude_opus_4_7", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20260416_introducing_claude_opus_4_7" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "revision": { "id": "revision_candidate_anthropic_20260416_introducing_claude_opus_4_7", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-4-7" }, { "id": "source_candidate_anthropic_20260528_introducing_claude_opus_4_8", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20260528_introducing_claude_opus_4_8" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-28", "revision": { "id": "revision_candidate_anthropic_20260528_introducing_claude_opus_4_8", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-4-8" }, { "id": "source_candidate_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20260609_claude_fable_5_and_claude_mythos_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "revision": { "id": "revision_candidate_anthropic_20260609_claude_fable_5_and_claude_mythos_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-fable-5-mythos-5" }, { "id": "source_candidate_anthropic_20260630_introducing_claude_sonnet_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20260630_introducing_claude_sonnet_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "revision": { "id": "revision_candidate_anthropic_20260630_introducing_claude_sonnet_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-sonnet-5" }, { "id": "source_candidate_anthropic_20260630_redeploying_fable_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20260630_redeploying_fable_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "revision": { "id": "revision_candidate_anthropic_20260630_redeploying_fable_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/redeploying-fable-5" }, { "id": "source_candidate_anthropic_20260724_introducing_claude_opus_5", "lab_id": "anthropic", "provenance": { "record_ids": [ "source_candidate_anthropic_20260724_introducing_claude_opus_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-24", "revision": { "id": "revision_candidate_anthropic_20260724_introducing_claude_opus_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://www.anthropic.com/news/claude-opus-5" }, { "id": "source_candidate_deepseek_20240517_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20240517_deepseek_chat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-17", "revision": { "id": "revision_candidate_deepseek_20240517_deepseek_chat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/updates" }, { "id": "source_candidate_deepseek_20240614_deepseek_coder", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20240614_deepseek_coder" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-14", "revision": { "id": "revision_candidate_deepseek_20240614_deepseek_coder", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/updates" }, { "id": "source_candidate_deepseek_20240628_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20240628_deepseek_chat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "revision": { "id": "revision_candidate_deepseek_20240628_deepseek_chat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/updates" }, { "id": "source_candidate_deepseek_20240724_deepseek_coder", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20240724_deepseek_coder" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-24", "revision": { "id": "revision_candidate_deepseek_20240724_deepseek_coder", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/updates" }, { "id": "source_candidate_deepseek_20240725_new_api_features", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20240725_new_api_features" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-25", "revision": { "id": "revision_candidate_deepseek_20240725_new_api_features", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news0725" }, { "id": "source_candidate_deepseek_20240802_api_launches_context_caching_on_disk_technology", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20240802_api_launches_context_caching_on_disk_technology" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-02", "revision": { "id": "revision_candidate_deepseek_20240802_api_launches_context_caching_on_disk_technology", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news0802" }, { "id": "source_candidate_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "revision": { "id": "revision_candidate_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news0905" }, { "id": "source_candidate_deepseek_20241120_deepseek_r1_lite_preview", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20241120_deepseek_r1_lite_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-20", "revision": { "id": "revision_candidate_deepseek_20241120_deepseek_r1_lite_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news1120" }, { "id": "source_candidate_deepseek_20241210_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20241210_deepseek_chat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "revision": { "id": "revision_candidate_deepseek_20241210_deepseek_chat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news1210" }, { "id": "source_candidate_deepseek_20241226_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20241226_deepseek_chat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "revision": { "id": "revision_candidate_deepseek_20241226_deepseek_chat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news1226" }, { "id": "source_candidate_deepseek_20250120_deepseek_reasoner", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20250120_deepseek_reasoner" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "revision": { "id": "revision_candidate_deepseek_20250120_deepseek_reasoner", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250120" }, { "id": "source_candidate_deepseek_20250324_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20250324_deepseek_chat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "revision": { "id": "revision_candidate_deepseek_20250324_deepseek_chat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250325" }, { "id": "source_candidate_deepseek_20250528_deepseek_reasoner", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20250528_deepseek_reasoner" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "revision": { "id": "revision_candidate_deepseek_20250528_deepseek_reasoner", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250528" }, { "id": "source_candidate_deepseek_20250821_deepseek_v3_1", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20250821_deepseek_v3_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "revision": { "id": "revision_candidate_deepseek_20250821_deepseek_v3_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250821" }, { "id": "source_candidate_deepseek_20250922_deepseek_v3_1_terminus", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_deepseek_20250922_deepseek_v3_1_terminus", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250922" }, { "id": "source_candidate_deepseek_20250929_deepseek_v3_2_exp", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "revision": { "id": "revision_candidate_deepseek_20250929_deepseek_v3_2_exp", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250929" }, { "id": "source_candidate_deepseek_20251201_deepseek_v3_2", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20251201_deepseek_v3_2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "revision": { "id": "revision_candidate_deepseek_20251201_deepseek_v3_2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news251201" }, { "id": "source_candidate_deepseek_20251201_deepseek_v3_2_speciale", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "revision": { "id": "revision_candidate_deepseek_20251201_deepseek_v3_2_speciale", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news251201" }, { "id": "source_candidate_deepseek_20260424_deepseek_v4", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20260424_deepseek_v4" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "revision": { "id": "revision_candidate_deepseek_20260424_deepseek_v4", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news260424" }, { "id": "source_candidate_deepseek_20260731_deepseek_v4_flash_update", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "revision": { "id": "revision_candidate_deepseek_20260731_deepseek_v4_flash_update", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/updates" }, { "id": "source_candidate_deepseek_20260813_deepseek_v4_pro_update", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "revision": { "id": "revision_candidate_deepseek_20260813_deepseek_v4_pro_update", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news260813" }, { "id": "source_candidate_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_candidate_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "revision": { "id": "revision_candidate_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news260821" }, { "id": "source_candidate_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20240208_the_next_chapter_of_our_gemini_era" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-08", "revision": { "id": "revision_candidate_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/ai/google-gemini-update-sundar-pichai-2024" }, { "id": "source_candidate_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20240215_our_next_generation_model_gemini_1_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "revision": { "id": "revision_candidate_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/products/google-gemini-next-generation-model-february-2024" }, { "id": "source_candidate_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "revision": { "id": "revision_candidate_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/gemma-open-models" }, { "id": "source_candidate_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "revision": { "id": "revision_candidate_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/products/google-gemini-update-flash-ai-assistant-io-2024" }, { "id": "source_candidate_google_deepmind_20240731_gemma_scope_helping_the_safety_community_shed_light", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20240731_gemma_scope_helping_the_safety_community_shed_light" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "revision": { "id": "revision_candidate_google_deepmind_20240731_gemma_scope_helping_the_safety_community_shed_light", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/gemma-scope-helping-the-safety-community-shed-light-on-the-inner-workings-of-language-models" }, { "id": "source_candidate_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "revision": { "id": "revision_candidate_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/updated-gemini-models-reduced-15-pro-pricing-increased-rate-limits-and-more" }, { "id": "source_candidate_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "revision": { "id": "revision_candidate_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/google-gemini-ai-update-december-2024" }, { "id": "source_candidate_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "revision": { "id": "revision_candidate_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-updates-february-2025" }, { "id": "source_candidate_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "revision": { "id": "revision_candidate_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/start-building-with-the-gemini-2-0-flash-family" }, { "id": "source_candidate_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "revision": { "id": "revision_candidate_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/experiment-with-gemini-20-flash-native-image-generation" }, { "id": "source_candidate_google_deepmind_20250312_gemini_robotics_brings_ai_into_the_physical_world", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250312_gemini_robotics_brings_ai_into_the_physical_world" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "revision": { "id": "revision_candidate_google_deepmind_20250312_gemini_robotics_brings_ai_into_the_physical_world", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/gemini-robotics-brings-ai-into-the-physical-world" }, { "id": "source_candidate_google_deepmind_20250312_introducing_gemma_3", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "revision": { "id": "revision_candidate_google_deepmind_20250312_introducing_gemma_3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/gemma-3" }, { "id": "source_candidate_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "revision": { "id": "revision_candidate_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025" }, { "id": "source_candidate_google_deepmind_20250414_dolphingemma_how_google_ai_is_helping_decode_dolphin", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250414_dolphingemma_how_google_ai_is_helping_decode_dolphin" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "revision": { "id": "revision_candidate_google_deepmind_20250414_dolphingemma_how_google_ai_is_helping_decode_dolphin", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/products/dolphingemma" }, { "id": "source_candidate_google_deepmind_20250415_generate_videos_in_gemini_and_whisk_with_veo_2", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250415_generate_videos_in_gemini_and_whisk_with_veo_2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-15", "revision": { "id": "revision_candidate_google_deepmind_20250415_generate_videos_in_gemini_and_whisk_with_veo_2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/video-generation" }, { "id": "source_candidate_google_deepmind_20250417_introducing_gemini_2_5_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "revision": { "id": "revision_candidate_google_deepmind_20250417_introducing_gemini_2_5_flash", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/start-building-with-gemini-25-flash" }, { "id": "source_candidate_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "revision": { "id": "revision_candidate_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/gemini-2-5-pro-updates" }, { "id": "source_candidate_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "revision": { "id": "revision_candidate_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/gemini-2-5-pro-io-improved-coding-performance" }, { "id": "source_candidate_google_deepmind_20250514_alphaevolve_a_gemini_powered_coding_agent_for_design", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250514_alphaevolve_a_gemini_powered_coding_agent_for_design" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-14", "revision": { "id": "revision_candidate_google_deepmind_20250514_alphaevolve_a_gemini_powered_coding_agent_for_design", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms" }, { "id": "source_candidate_google_deepmind_20250520_advancing_gemini_s_security_safeguards", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250520_advancing_gemini_s_security_safeguards" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "revision": { "id": "revision_candidate_google_deepmind_20250520_advancing_gemini_s_security_safeguards", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/advancing-geminis-security-safeguards" }, { "id": "source_candidate_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "revision": { "id": "revision_candidate_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/introducing-gemma-3n" }, { "id": "source_candidate_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "revision": { "id": "revision_candidate_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/google-gemini-updates-io-2025" }, { "id": "source_candidate_google_deepmind_20250603_advanced_audio_dialog_and_generation_with_gemini_2_5", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250603_advanced_audio_dialog_and_generation_with_gemini_2_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-03", "revision": { "id": "revision_candidate_google_deepmind_20250603_advanced_audio_dialog_and_generation_with_gemini_2_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-2-5-native-audio" }, { "id": "source_candidate_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "revision": { "id": "revision_candidate_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/gemini-2-5-thinking-model-updates" }, { "id": "source_candidate_google_deepmind_20250617_we_re_expanding_our_gemini_2_5_family_of_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250617_we_re_expanding_our_gemini_2_5_family_of_models" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "revision": { "id": "revision_candidate_google_deepmind_20250617_we_re_expanding_our_gemini_2_5_family_of_models", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/gemini-2-5-model-family-expands" }, { "id": "source_candidate_google_deepmind_20250624_gemini_robotics_on_device_brings_ai_to_local_robotic", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250624_gemini_robotics_on_device_brings_ai_to_local_robotic" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-24", "revision": { "id": "revision_candidate_google_deepmind_20250624_gemini_robotics_on_device_brings_ai_to_local_robotic", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/gemini-robotics-on-device-brings-ai-to-local-robotic-devices" }, { "id": "source_candidate_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "revision": { "id": "revision_candidate_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/introducing-gemma-3n-developer-guide" }, { "id": "source_candidate_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "revision": { "id": "revision_candidate_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://research.google/blog/medgemma-our-most-capable-open-models-for-health-ai-development" }, { "id": "source_candidate_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "revision": { "id": "revision_candidate_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/t5gemma" }, { "id": "source_candidate_google_deepmind_20250721_advanced_version_of_gemini_with_deep_think_officiall", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250721_advanced_version_of_gemini_with_deep_think_officiall" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-21", "revision": { "id": "revision_candidate_google_deepmind_20250721_advanced_version_of_gemini_with_deep_think_officiall", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/advanced-version-of-gemini-with-deep-think-officially-achieves-gold-medal-standard-at-the-international-mathematical-olympiad" }, { "id": "source_candidate_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "revision": { "id": "revision_candidate_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/gemini-25-flash-lite-is-now-stable-and-generally-available" }, { "id": "source_candidate_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250801_try_deep_think_in_the_gemini_app" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "revision": { "id": "revision_candidate_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/gemini-2-5-deep-think" }, { "id": "source_candidate_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "revision": { "id": "revision_candidate_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/introducing-gemma-3-270m" }, { "id": "source_candidate_google_deepmind_20250826_image_editing_in_gemini_just_got_a_major_upgrade", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250826_image_editing_in_gemini_just_got_a_major_upgrade" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-26", "revision": { "id": "revision_candidate_google_deepmind_20250826_image_editing_in_gemini_just_got_a_major_upgrade", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/updated-image-editing-model" }, { "id": "source_candidate_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "revision": { "id": "revision_candidate_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://research.google/blog/vaultgemma-the-worlds-most-capable-differentially-private-llm" }, { "id": "source_candidate_google_deepmind_20250917_gemini_achieves_gold_medal_level_at_the_internationa", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250917_gemini_achieves_gold_medal_level_at_the_internationa" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-17", "revision": { "id": "revision_candidate_google_deepmind_20250917_gemini_achieves_gold_medal_level_at_the_internationa", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/gemini-achieves-gold-medal-level-at-the-international-collegiate-programming-contest-world-finals" }, { "id": "source_candidate_google_deepmind_20250925_gemini_robotics_1_5_brings_ai_agents_into_the_physic", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20250925_gemini_robotics_1_5_brings_ai_agents_into_the_physic" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-25", "revision": { "id": "revision_candidate_google_deepmind_20250925_gemini_robotics_1_5_brings_ai_agents_into_the_physic", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/gemini-robotics-15-brings-ai-agents-into-the-physical-world" }, { "id": "source_candidate_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "revision": { "id": "revision_candidate_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-computer-use-model" }, { "id": "source_candidate_google_deepmind_20251015_how_a_gemma_model_helped_discover_a_new_potential_ca", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20251015_how_a_gemma_model_helped_discover_a_new_potential_ca" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-15", "revision": { "id": "revision_candidate_google_deepmind_20251015_how_a_gemma_model_helped_discover_a_new_potential_ca", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/products/google-gemma-ai-cancer-therapy-discovery" }, { "id": "source_candidate_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "revision": { "id": "revision_candidate_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3" }, { "id": "source_candidate_google_deepmind_20251118_start_building_with_gemini_3", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20251118_start_building_with_gemini_3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "revision": { "id": "revision_candidate_google_deepmind_20251118_start_building_with_gemini_3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/gemini-3-developers" }, { "id": "source_candidate_google_deepmind_20251120_build_with_nano_banana_pro_our_gemini_3_pro_image_mo", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20251120_build_with_nano_banana_pro_our_gemini_3_pro_image_mo" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-20", "revision": { "id": "revision_candidate_google_deepmind_20251120_build_with_nano_banana_pro_our_gemini_3_pro_image_mo", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/gemini-3-pro-image-developers" }, { "id": "source_candidate_google_deepmind_20251120_how_we_re_bringing_ai_image_verification_to_the_gemi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20251120_how_we_re_bringing_ai_image_verification_to_the_gemi" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-20", "revision": { "id": "revision_candidate_google_deepmind_20251120_how_we_re_bringing_ai_image_verification_to_the_gemi", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/products/ai-image-verification-gemini-app" }, { "id": "source_candidate_google_deepmind_20251212_improved_gemini_audio_models_for_powerful_voice_expe", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20251212_improved_gemini_audio_models_for_powerful_voice_expe" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-12", "revision": { "id": "revision_candidate_google_deepmind_20251212_improved_gemini_audio_models_for_powerful_voice_expe", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/gemini-audio-model-updates" }, { "id": "source_candidate_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "revision": { "id": "revision_candidate_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3-flash" }, { "id": "source_candidate_google_deepmind_20251219_gemma_scope_2_helping_the_ai_safety_community_deepen", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20251219_gemma_scope_2_helping_the_ai_safety_community_deepen" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-19", "revision": { "id": "revision_candidate_google_deepmind_20251219_gemma_scope_2_helping_the_ai_safety_community_deepen", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/gemma-scope-2-helping-the-ai-safety-community-deepen-understanding-of-complex-language-model-behavior" }, { "id": "source_candidate_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-11", "revision": { "id": "revision_candidate_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/accelerating-mathematical-and-scientific-discovery-with-gemini-deep-think" }, { "id": "source_candidate_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "revision": { "id": "revision_candidate_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-deep-think" }, { "id": "source_candidate_google_deepmind_20260218_a_new_way_to_express_yourself_gemini_can_now_create", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260218_a_new_way_to_express_yourself_gemini_can_now_create" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-18", "revision": { "id": "revision_candidate_google_deepmind_20260218_a_new_way_to_express_yourself_gemini_can_now_create", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/products/gemini-app/lyria-3" }, { "id": "source_candidate_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "revision": { "id": "revision_candidate_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro" }, { "id": "source_candidate_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "revision": { "id": "revision_candidate_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-flash-lite" }, { "id": "source_candidate_google_deepmind_20260326_gemini_3_1_flash_live_making_audio_ai_more_natural_a", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260326_gemini_3_1_flash_live_making_audio_ai_more_natural_a" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-26", "revision": { "id": "revision_candidate_google_deepmind_20260326_gemini_3_1_flash_live_making_audio_ai_more_natural_a", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-flash-live" }, { "id": "source_candidate_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "revision": { "id": "revision_candidate_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4" }, { "id": "source_candidate_google_deepmind_20260414_gemini_robotics_er_1_6_powering_real_world_robotics", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260414_gemini_robotics_er_1_6_powering_real_world_robotics" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-14", "revision": { "id": "revision_candidate_google_deepmind_20260414_gemini_robotics_er_1_6_powering_real_world_robotics", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/gemini-robotics-er-1-6" }, { "id": "source_candidate_google_deepmind_20260415_gemini_3_1_flash_tts_the_next_generation_of_expressi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260415_gemini_3_1_flash_tts_the_next_generation_of_expressi" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "revision": { "id": "revision_candidate_google_deepmind_20260415_gemini_3_1_flash_tts_the_next_generation_of_expressi", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-flash-tts" }, { "id": "source_candidate_google_deepmind_20260507_alphaevolve_how_our_gemini_powered_coding_agent_is_s", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260507_alphaevolve_how_our_gemini_powered_coding_agent_is_s" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-07", "revision": { "id": "revision_candidate_google_deepmind_20260507_alphaevolve_how_our_gemini_powered_coding_agent_is_s", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/alphaevolve-impact" }, { "id": "source_candidate_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "revision": { "id": "revision_candidate_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5" }, { "id": "source_candidate_google_deepmind_20260519_gemini_for_science_ai_experiments_and_tools_for_a_ne", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260519_gemini_for_science_ai_experiments_and_tools_for_a_ne" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "revision": { "id": "revision_candidate_google_deepmind_20260519_gemini_for_science_ai_experiments_and_tools_for_a_ne", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/research/gemini-for-science-io-2026" }, { "id": "source_candidate_google_deepmind_20260519_introducing_gemini_omni", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260519_introducing_gemini_omni" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "revision": { "id": "revision_candidate_google_deepmind_20260519_introducing_gemini_omni", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-omni" }, { "id": "source_candidate_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "revision": { "id": "revision_candidate_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/introducing-gemma-4-12b" }, { "id": "source_candidate_google_deepmind_20260609_fluid_natural_voice_translation_with_gemini_3_5_live", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260609_fluid_natural_voice_translation_with_gemini_3_5_live" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-09", "revision": { "id": "revision_candidate_google_deepmind_20260609_fluid_natural_voice_translation_with_gemini_3_5_live", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-live-3-5-translate" }, { "id": "source_candidate_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-10", "revision": { "id": "revision_candidate_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/diffusion-gemma-faster-text-generation" }, { "id": "source_candidate_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-24", "revision": { "id": "revision_candidate_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-computer-use-gemini-3-5-flash" }, { "id": "source_candidate_google_deepmind_20260630_start_building_with_nano_banana_2_lite_and_gemini_om", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260630_start_building_with_nano_banana_2_lite_and_gemini_om" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-30", "revision": { "id": "revision_candidate_google_deepmind_20260630_start_building_with_nano_banana_2_lite_and_gemini_om", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-omni-flash-nano-banana-2-lite" }, { "id": "source_candidate_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "revision": { "id": "revision_candidate_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/introducing-gemini-3-5-flash-cyber" }, { "id": "source_candidate_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "revision": { "id": "revision_candidate_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-6-flash-3-5-flash-lite-3-5-flash-cyber" }, { "id": "source_candidate_google_deepmind_20260730_gemini_robotics_2_brings_whole_body_intelligence_to", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260730_gemini_robotics_2_brings_whole_body_intelligence_to" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "revision": { "id": "revision_candidate_google_deepmind_20260730_gemini_robotics_2_brings_whole_body_intelligence_to", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/gemini-robotics-2-brings-whole-body-intelligence-to-robots" }, { "id": "source_candidate_google_deepmind_20260730_gemini_robotics_er_2_powering_robotics_with_video_un", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260730_gemini_robotics_er_2_powering_robotics_with_video_un" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "revision": { "id": "revision_candidate_google_deepmind_20260730_gemini_robotics_er_2_powering_robotics_with_video_un", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-robotics-er-2" }, { "id": "source_candidate_google_deepmind_20260813_introducing_gemini_3_7_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "revision": { "id": "revision_candidate_google_deepmind_20260813_introducing_gemini_3_7_flash", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-3-7-flash" }, { "id": "source_candidate_google_deepmind_20260826_intelligent_transcription_with_gemini_3_5_transcribe", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260826_intelligent_transcription_with_gemini_3_5_transcribe" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "revision": { "id": "revision_candidate_google_deepmind_20260826_intelligent_transcription_with_gemini_3_5_transcribe", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5-transcribe" }, { "id": "source_candidate_google_deepmind_20260827_gemini_omni_1_1_flash_lets_you_build_with_more_contr", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_candidate_google_deepmind_20260827_gemini_omni_1_1_flash_lets_you_build_with_more_contr" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-27", "revision": { "id": "revision_candidate_google_deepmind_20260827_gemini_omni_1_1_flash_lets_you_build_with_more_contr", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/build-with-gemini-omni-1-1-flash" }, { "id": "source_candidate_meta_20240215_v_jepa", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20240215_v_jepa" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "revision": { "id": "revision_candidate_meta_20240215_v_jepa", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/v-jepa-yann-lecun-ai-model-video-joint-embedding-predictive-architecture/" }, { "id": "source_candidate_meta_20240320_scenescript", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20240320_scenescript" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-20", "revision": { "id": "revision_candidate_meta_20240320_scenescript", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/scenescript-3d-scene-reconstruction-reality-labs-research/" }, { "id": "source_candidate_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "revision": { "id": "revision_candidate_meta_20240418_introducing_meta_llama_3_the_most_capable_openly_ava", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-llama-3/" }, { "id": "source_candidate_meta_20240418_our_responsible_approach_to_meta_ai_and_meta_llama_3", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20240418_our_responsible_approach_to_meta_ai_and_meta_llama_3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "revision": { "id": "revision_candidate_meta_20240418_our_responsible_approach_to_meta_ai_and_meta_llama_3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-llama-3-meta-ai-responsibility/" }, { "id": "source_candidate_meta_20240618_meta_fair_research_new_releases", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20240618_meta_fair_research_new_releases" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "revision": { "id": "revision_candidate_meta_20240618_meta_fair_research_new_releases", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-fair-research-new-releases/" }, { "id": "source_candidate_meta_20240723_expanding_our_open_source_large_language_models_resp", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20240723_expanding_our_open_source_large_language_models_resp" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "revision": { "id": "revision_candidate_meta_20240723_expanding_our_open_source_large_language_models_resp", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-llama-3-1-ai-responsibility/" }, { "id": "source_candidate_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "revision": { "id": "revision_candidate_meta_20240723_introducing_llama_3_1_our_most_capable_models_to_dat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-llama-3-1/" }, { "id": "source_candidate_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "revision": { "id": "revision_candidate_meta_20240925_llama_3_2_revolutionizing_edge_ai_and_vision_with_op", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/llama-3-2-connect-2024-vision-edge-mobile-devices/" }, { "id": "source_candidate_meta_20241004_meta_movie_gen", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20241004_meta_movie_gen" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-04", "revision": { "id": "revision_candidate_meta_20241004_meta_movie_gen", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/movie-gen-media-foundation-models-generative-ai-video/" }, { "id": "source_candidate_meta_20241018_meta_fair_research_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20241018_meta_fair_research_bundle" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "revision": { "id": "revision_candidate_meta_20241018_meta_fair_research_bundle", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/fair-news-segment-anything-2-1-meta-spirit-lm-layer-skip-salsa-lingua/" }, { "id": "source_candidate_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20241024_introducing_quantized_llama_models_with_increased_sp" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "revision": { "id": "revision_candidate_meta_20241024_introducing_quantized_llama_models_with_increased_sp", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-llama-quantized-lightweight-models/" }, { "id": "source_candidate_meta_20241031_embodied_ai_robotics_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20241031_embodied_ai_robotics_bundle" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-31", "revision": { "id": "revision_candidate_meta_20241031_embodied_ai_robotics_bundle", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/fair-robotics-open-source/" }, { "id": "source_candidate_meta_20241212_meta_fair_agents_architecture_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20241212_meta_fair_agents_architecture_bundle" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "revision": { "id": "revision_candidate_meta_20241212_meta_fair_agents_architecture_bundle", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-fair-updates-agents-robustness-safety-architecture/" }, { "id": "source_candidate_meta_20241219_the_future_of_ai_built_with_llama", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20241219_the_future_of_ai_built_with_llama" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "revision": { "id": "revision_candidate_meta_20241219_the_future_of_ai_built_with_llama", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/future-of-ai-built-with-llama/" }, { "id": "source_candidate_meta_20250207_human_centered_research_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20250207_human_centered_research_bundle" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-07", "revision": { "id": "revision_candidate_meta_20250207_human_centered_research_bundle", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/machine-intelligence-research-new-models/" }, { "id": "source_candidate_meta_20250212_sam_2_1_sagemaker_update", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20250212_sam_2_1_sagemaker_update" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-12", "revision": { "id": "revision_candidate_meta_20250212_sam_2_1_sagemaker_update", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/segment-anything-2-video/" }, { "id": "source_candidate_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "revision": { "id": "revision_candidate_meta_20250405_the_llama_4_herd_the_beginning_of_a_new_era_of_nativ", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/" }, { "id": "source_candidate_meta_20250417_perception_localization_reasoning_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20250417_perception_localization_reasoning_bundle" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "revision": { "id": "revision_candidate_meta_20250417_perception_localization_reasoning_bundle", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-fair-updates-perception-localization-reasoning/" }, { "id": "source_candidate_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20250429_everything_we_announced_at_our_first_ever_llamacon" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "revision": { "id": "revision_candidate_meta_20250429_everything_we_announced_at_our_first_ever_llamacon", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/llamacon-llama-news/" }, { "id": "source_candidate_meta_20250429_llama_protection_tools", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20250429_llama_protection_tools" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "revision": { "id": "revision_candidate_meta_20250429_llama_protection_tools", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/ai-defenders-program-llama-protection-tools/" }, { "id": "source_candidate_meta_20250514_fair_science_release_bundle", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20250514_fair_science_release_bundle" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-14", "revision": { "id": "revision_candidate_meta_20250514_fair_science_release_bundle", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-fair-science-new-open-source-releases/" }, { "id": "source_candidate_meta_20250611_v_jepa_2", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20250611_v_jepa_2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-11", "revision": { "id": "revision_candidate_meta_20250611_v_jepa_2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/v-jepa-2-world-model-benchmarks/" }, { "id": "source_candidate_meta_20250627_seamless_interaction", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20250627_seamless_interaction" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-27", "revision": { "id": "revision_candidate_meta_20250627_seamless_interaction", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/seamless-interaction-natural-conversational-dynamics/" }, { "id": "source_candidate_meta_20250814_dinov3", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20250814_dinov3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "revision": { "id": "revision_candidate_meta_20250814_dinov3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/dinov3-self-supervised-vision-model/" }, { "id": "source_candidate_meta_20251110_omnilingual_asr", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20251110_omnilingual_asr" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-10", "revision": { "id": "revision_candidate_meta_20251110_omnilingual_asr", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/omnilingual-asr-advancing-automatic-speech-recognition/" }, { "id": "source_candidate_meta_20251119_sam_3d", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20251119_sam_3d" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "revision": { "id": "revision_candidate_meta_20251119_sam_3d", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/sam-3d/" }, { "id": "source_candidate_meta_20251216_sam_audio", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20251216_sam_audio" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-16", "revision": { "id": "revision_candidate_meta_20251216_sam_audio", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/sam-audio/" }, { "id": "source_candidate_meta_20260327_sam_3_1", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20260327_sam_3_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-27", "revision": { "id": "revision_candidate_meta_20260327_sam_3_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/segment-anything-model-3/" }, { "id": "source_candidate_meta_20260408_introducing_muse_spark", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20260408_introducing_muse_spark" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "revision": { "id": "revision_candidate_meta_20260408_introducing_muse_spark", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/introducing-muse-spark-msl/" }, { "id": "source_candidate_meta_20260629_brain2qwerty", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20260629_brain2qwerty" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-29", "revision": { "id": "revision_candidate_meta_20260629_brain2qwerty", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/brain2qwerty-brain-ai-human-communication/" }, { "id": "source_candidate_meta_20260707_introducing_muse_image_and_muse_video", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20260707_introducing_muse_image_and_muse_video" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-07", "revision": { "id": "revision_candidate_meta_20260707_introducing_muse_image_and_muse_video", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/introducing-muse-image-muse-video-msl/" }, { "id": "source_candidate_meta_20260709_introducing_muse_spark_1_1", "lab_id": "meta", "provenance": { "record_ids": [ "source_candidate_meta_20260709_introducing_muse_spark_1_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "revision": { "id": "revision_candidate_meta_20260709_introducing_muse_spark_1_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/" }, { "id": "source_candidate_openai_20240125_new_embedding_models_and_api_updates", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20240125_new_embedding_models_and_api_updates" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "revision": { "id": "revision_candidate_openai_20240125_new_embedding_models_and_api_updates", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/new-embedding-models-and-api-updates/" }, { "id": "source_candidate_openai_20240513_hello_gpt_4o", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20240513_hello_gpt_4o" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-13", "revision": { "id": "revision_candidate_openai_20240513_hello_gpt_4o", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/hello-gpt-4o/" }, { "id": "source_candidate_openai_20240513_introducing_gpt_4o_and_more_tools_to_chatgpt_free_users", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20240513_introducing_gpt_4o_and_more_tools_to_chatgpt_free_users" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-13", "revision": { "id": "revision_candidate_openai_20240513_introducing_gpt_4o_and_more_tools_to_chatgpt_free_users", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-4o-and-more-tools-to-chatgpt-free/" }, { "id": "source_candidate_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "revision": { "id": "revision_candidate_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/" }, { "id": "source_candidate_openai_20240912_introducing_openai_o1_preview", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20240912_introducing_openai_o1_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "revision": { "id": "revision_candidate_openai_20240912_introducing_openai_o1_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-openai-o1-preview/" }, { "id": "source_candidate_openai_20240912_openai_o1_mini", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20240912_openai_o1_mini" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "revision": { "id": "revision_candidate_openai_20240912_openai_o1_mini", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/openai-o1-mini-advancing-cost-efficient-reasoning/" }, { "id": "source_candidate_openai_20240926_upgrading_the_moderation_api_with_our_new_multimodal_moderation_model", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20240926_upgrading_the_moderation_api_with_our_new_multimodal_moderation_model" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-26", "revision": { "id": "revision_candidate_openai_20240926_upgrading_the_moderation_api_with_our_new_multimodal_moderation_model", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/upgrading-the-moderation-api-with-our-new-multimodal-moderation-model/" }, { "id": "source_candidate_openai_20241001_introducing_the_realtime_api", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20241001_introducing_the_realtime_api" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-01", "revision": { "id": "revision_candidate_openai_20241001_introducing_the_realtime_api", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-the-realtime-api/" }, { "id": "source_candidate_openai_20241209_sora_is_here", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20241209_sora_is_here" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-09", "revision": { "id": "revision_candidate_openai_20241209_sora_is_here", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/sora-is-here/" }, { "id": "source_candidate_openai_20241217_openai_o1_and_new_tools_for_developers", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "revision": { "id": "revision_candidate_openai_20241217_openai_o1_and_new_tools_for_developers", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/o1-and-new-tools-for-developers/" }, { "id": "source_candidate_openai_20250123_computer_using_agent", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250123_computer_using_agent" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "revision": { "id": "revision_candidate_openai_20250123_computer_using_agent", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/computer-using-agent/" }, { "id": "source_candidate_openai_20250123_introducing_operator", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250123_introducing_operator" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "revision": { "id": "revision_candidate_openai_20250123_introducing_operator", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-operator/" }, { "id": "source_candidate_openai_20250131_openai_o3_mini", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250131_openai_o3_mini" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "revision": { "id": "revision_candidate_openai_20250131_openai_o3_mini", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/openai-o3-mini/" }, { "id": "source_candidate_openai_20250227_introducing_gpt_4_5", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250227_introducing_gpt_4_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "revision": { "id": "revision_candidate_openai_20250227_introducing_gpt_4_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-4-5/" }, { "id": "source_candidate_openai_20250320_introducing_next_generation_audio_models_in_the_api", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250320_introducing_next_generation_audio_models_in_the_api" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-20", "revision": { "id": "revision_candidate_openai_20250320_introducing_next_generation_audio_models_in_the_api", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-our-next-generation-audio-models/" }, { "id": "source_candidate_openai_20250325_introducing_4o_image_generation", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250325_introducing_4o_image_generation" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "revision": { "id": "revision_candidate_openai_20250325_introducing_4o_image_generation", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-4o-image-generation/" }, { "id": "source_candidate_openai_20250414_introducing_gpt_4_1_in_the_api", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "revision": { "id": "revision_candidate_openai_20250414_introducing_gpt_4_1_in_the_api", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-4-1/" }, { "id": "source_candidate_openai_20250416_introducing_openai_o3_and_o4_mini", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "revision": { "id": "revision_candidate_openai_20250416_introducing_openai_o3_and_o4_mini", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-o3-and-o4-mini/" }, { "id": "source_candidate_openai_20250423_introducing_our_latest_image_generation_model_in_the_api", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250423_introducing_our_latest_image_generation_model_in_the_api" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-23", "revision": { "id": "revision_candidate_openai_20250423_introducing_our_latest_image_generation_model_in_the_api", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/image-generation-api/" }, { "id": "source_candidate_openai_20250516_introducing_codex", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250516_introducing_codex" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-16", "revision": { "id": "revision_candidate_openai_20250516_introducing_codex", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-codex/" }, { "id": "source_candidate_openai_20250805_introducing_gpt_oss", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250805_introducing_gpt_oss" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "revision": { "id": "revision_candidate_openai_20250805_introducing_gpt_oss", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-oss/" }, { "id": "source_candidate_openai_20250807_introducing_gpt_5", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250807_introducing_gpt_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "revision": { "id": "revision_candidate_openai_20250807_introducing_gpt_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5/" }, { "id": "source_candidate_openai_20250807_introducing_gpt_5_for_developers", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250807_introducing_gpt_5_for_developers" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "revision": { "id": "revision_candidate_openai_20250807_introducing_gpt_5_for_developers", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-for-developers/" }, { "id": "source_candidate_openai_20250828_introducing_gpt_realtime_and_realtime_api_updates_fo", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250828_introducing_gpt_realtime_and_realtime_api_updates_fo" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-28", "revision": { "id": "revision_candidate_openai_20250828_introducing_gpt_realtime_and_realtime_api_updates_fo", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-realtime/" }, { "id": "source_candidate_openai_20250915_introducing_upgrades_to_codex", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250915_introducing_upgrades_to_codex" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "revision": { "id": "revision_candidate_openai_20250915_introducing_upgrades_to_codex", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-upgrades-to-codex/" }, { "id": "source_candidate_openai_20250930_sora_2_is_here", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20250930_sora_2_is_here" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-30", "revision": { "id": "revision_candidate_openai_20250930_sora_2_is_here", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/sora-2/" }, { "id": "source_candidate_openai_20251029_introducing_gpt_oss_safeguard", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20251029_introducing_gpt_oss_safeguard" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-29", "revision": { "id": "revision_candidate_openai_20251029_introducing_gpt_oss_safeguard", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-oss-safeguard/" }, { "id": "source_candidate_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "revision": { "id": "revision_candidate_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-1/" }, { "id": "source_candidate_openai_20251113_introducing_gpt_5_1_for_developers", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20251113_introducing_gpt_5_1_for_developers" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-13", "revision": { "id": "revision_candidate_openai_20251113_introducing_gpt_5_1_for_developers", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-1-for-developers/" }, { "id": "source_candidate_openai_20251119_building_more_with_gpt_5_1_codex_max", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "revision": { "id": "revision_candidate_openai_20251119_building_more_with_gpt_5_1_codex_max", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-1-codex-max/" }, { "id": "source_candidate_openai_20251211_advancing_science_and_math_with_gpt_5_2", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20251211_advancing_science_and_math_with_gpt_5_2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "revision": { "id": "revision_candidate_openai_20251211_advancing_science_and_math_with_gpt_5_2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-2-for-science-and-math/" }, { "id": "source_candidate_openai_20251211_introducing_gpt_5_2", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20251211_introducing_gpt_5_2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "revision": { "id": "revision_candidate_openai_20251211_introducing_gpt_5_2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-2/" }, { "id": "source_candidate_openai_20251216_the_new_chatgpt_images_is_here", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20251216_the_new_chatgpt_images_is_here" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-16", "revision": { "id": "revision_candidate_openai_20251216_the_new_chatgpt_images_is_here", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/new-chatgpt-images-is-here/" }, { "id": "source_candidate_openai_20251218_introducing_gpt_5_2_codex", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "revision": { "id": "revision_candidate_openai_20251218_introducing_gpt_5_2_codex", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-2-codex/" }, { "id": "source_candidate_openai_20260205_introducing_gpt_5_3_codex", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "revision": { "id": "revision_candidate_openai_20260205_introducing_gpt_5_3_codex", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-3-codex/" }, { "id": "source_candidate_openai_20260212_introducing_gpt_5_3_codex_spark", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260212_introducing_gpt_5_3_codex_spark" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "revision": { "id": "revision_candidate_openai_20260212_introducing_gpt_5_3_codex_spark", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-3-codex-spark/" }, { "id": "source_candidate_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "revision": { "id": "revision_candidate_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-3-instant/" }, { "id": "source_candidate_openai_20260305_introducing_gpt_5_4", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260305_introducing_gpt_5_4" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "revision": { "id": "revision_candidate_openai_20260305_introducing_gpt_5_4", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-4/" }, { "id": "source_candidate_openai_20260317_introducing_gpt_5_4_mini_and_nano", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "revision": { "id": "revision_candidate_openai_20260317_introducing_gpt_5_4_mini_and_nano", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/" }, { "id": "source_candidate_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "revision": { "id": "revision_candidate_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-rosalind/" }, { "id": "source_candidate_openai_20260421_introducing_chatgpt_images_2_0", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260421_introducing_chatgpt_images_2_0" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-21", "revision": { "id": "revision_candidate_openai_20260421_introducing_chatgpt_images_2_0", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-chatgpt-images-2-0/" }, { "id": "source_candidate_openai_20260423_introducing_gpt_5_5", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260423_introducing_gpt_5_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "revision": { "id": "revision_candidate_openai_20260423_introducing_gpt_5_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-5/" }, { "id": "source_candidate_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "revision": { "id": "revision_candidate_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-5-instant/" }, { "id": "source_candidate_openai_20260507_advancing_voice_intelligence_with_new_models_in_the_api", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260507_advancing_voice_intelligence_with_new_models_in_the_api" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-07", "revision": { "id": "revision_candidate_openai_20260507_advancing_voice_intelligence_with_new_models_in_the_api", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/advancing-voice-intelligence-with-new-models-in-the-api/" }, { "id": "source_candidate_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "revision": { "id": "revision_candidate_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-new-capabilities-to-gpt-rosalind/" }, { "id": "source_candidate_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "revision": { "id": "revision_candidate_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/previewing-gpt-5-6-sol/" }, { "id": "source_candidate_openai_20260708_introducing_gpt_live", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260708_introducing_gpt_live" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-08", "revision": { "id": "revision_candidate_openai_20260708_introducing_gpt_live", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-live/" }, { "id": "source_candidate_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "revision": { "id": "revision_candidate_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-6/" }, { "id": "source_candidate_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "revision": { "id": "revision_candidate_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/advancing-the-price-performance-frontier-with-gpt-5-6/" }, { "id": "source_candidate_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "revision": { "id": "revision_candidate_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/improving-gpt-5-6-sol-in-chatgpt/" }, { "id": "source_candidate_openai_20260811_daybreak_models_are_now_available_on_aws", "lab_id": "openai", "provenance": { "record_ids": [ "source_candidate_openai_20260811_daybreak_models_are_now_available_on_aws" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-11", "revision": { "id": "revision_candidate_openai_20260811_daybreak_models_are_now_available_on_aws", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://openai.com/index/daybreak-models-are-now-available-on-aws/" }, { "id": "source_candidate_qwen_20240123_introducing_qwen_qwen", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240123_introducing_qwen_qwen" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "revision": { "id": "revision_candidate_qwen_20240123_introducing_qwen_qwen", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen" }, { "id": "source_candidate_qwen_20240125_introducing_qwen_vl_qwen_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "revision": { "id": "revision_candidate_qwen_20240125_introducing_qwen_vl_qwen_vl", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-vl" }, { "id": "source_candidate_qwen_20240204_introducing_qwen1_5_qwen1_5", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "revision": { "id": "revision_candidate_qwen_20240204_introducing_qwen1_5_qwen1_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen1.5" }, { "id": "source_candidate_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "revision": { "id": "revision_candidate_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-moe" }, { "id": "source_candidate_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "revision": { "id": "revision_candidate_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen1.5-32b" }, { "id": "source_candidate_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "revision": { "id": "revision_candidate_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/codeqwen1.5" }, { "id": "source_candidate_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "revision": { "id": "revision_candidate_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen1.5-110b" }, { "id": "source_candidate_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-11", "revision": { "id": "revision_candidate_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-max-0428" }, { "id": "source_candidate_qwen_20240606_generalizing_an_llm_from_8k_to_1m_context_using_qwen_qwen_agent_2405", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240606_generalizing_an_llm_from_8k_to_1m_context_using_qwen_qwen_agent_2405" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "revision": { "id": "revision_candidate_qwen_20240606_generalizing_an_llm_from_8k_to_1m_context_using_qwen_qwen_agent_2405", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-agent-2405" }, { "id": "source_candidate_qwen_20240606_hello_qwen2_qwen2", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "revision": { "id": "revision_candidate_qwen_20240606_hello_qwen2_qwen2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2" }, { "id": "source_candidate_qwen_20240807_introducing_qwen2_math_qwen2_math", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "revision": { "id": "revision_candidate_qwen_20240807_introducing_qwen2_math_qwen2_math", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2-math" }, { "id": "source_candidate_qwen_20240809_qwen2_audio_chat_with_your_voice_qwen2_audio", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240809_qwen2_audio_chat_with_your_voice_qwen2_audio" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-09", "revision": { "id": "revision_candidate_qwen_20240809_qwen2_audio_chat_with_your_voice_qwen2_audio", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2-audio" }, { "id": "source_candidate_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "revision": { "id": "revision_candidate_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2-vl" }, { "id": "source_candidate_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "revision": { "id": "revision_candidate_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5" }, { "id": "source_candidate_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "revision": { "id": "revision_candidate_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-coder" }, { "id": "source_candidate_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "revision": { "id": "revision_candidate_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-llm" }, { "id": "source_candidate_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "revision": { "id": "revision_candidate_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-math" }, { "id": "source_candidate_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "revision": { "id": "revision_candidate_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-coder-family" }, { "id": "source_candidate_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "revision": { "id": "revision_candidate_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-turbo" }, { "id": "source_candidate_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "revision": { "id": "revision_candidate_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwq-32b-preview" }, { "id": "source_candidate_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "revision": { "id": "revision_candidate_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qvq-72b-preview" }, { "id": "source_candidate_qwen_20250113_towards_effective_process_supervision_in_mathematica_qwen2_5_math_prm", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250113_towards_effective_process_supervision_in_mathematica_qwen2_5_math_prm" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-13", "revision": { "id": "revision_candidate_qwen_20250113_towards_effective_process_supervision_in_mathematica_qwen2_5_math_prm", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-math-prm" }, { "id": "source_candidate_qwen_20250120_global_batch_load_balance_almost_free_lunch_to_impro_global_load_balance", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250120_global_batch_load_balance_almost_free_lunch_to_impro_global_load_balance" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "revision": { "id": "revision_candidate_qwen_20250120_global_batch_load_balance_almost_free_lunch_to_impro_global_load_balance", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/global-load-balance" }, { "id": "source_candidate_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "revision": { "id": "revision_candidate_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-1m" }, { "id": "source_candidate_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "revision": { "id": "revision_candidate_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-vl" }, { "id": "source_candidate_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "revision": { "id": "revision_candidate_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-max" }, { "id": "source_candidate_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "revision": { "id": "revision_candidate_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwq-max-preview" }, { "id": "source_candidate_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "revision": { "id": "revision_candidate_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwq-32b" }, { "id": "source_candidate_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "revision": { "id": "revision_candidate_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-vl-32b" }, { "id": "source_candidate_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "revision": { "id": "revision_candidate_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-omni" }, { "id": "source_candidate_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-27", "revision": { "id": "revision_candidate_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qvq-max-preview" }, { "id": "source_candidate_qwen_20250428_qwen3_think_deeper_act_faster_dbf896e521a5dc531a9b9cdd74f487d8908569fe_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250428_qwen3_think_deeper_act_faster_dbf896e521a5dc531a9b9cdd74f487d8908569fe_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "revision": { "id": "revision_candidate_qwen_20250428_qwen3_think_deeper_act_faster_dbf896e521a5dc531a9b9cdd74f487d8908569fe_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/dbf896e521a5dc531a9b9cdd74f487d8908569fe" }, { "id": "source_candidate_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "revision": { "id": "revision_candidate_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3" }, { "id": "source_candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_64e75147c1b7c0a4f63f54113a11594ae10c6142_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_64e75147c1b7c0a4f63f54113a11594ae10c6142_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-05", "revision": { "id": "revision_candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_64e75147c1b7c0a4f63f54113a11594ae10c6142_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/64e75147c1b7c0a4f63f54113a11594ae10c6142" }, { "id": "source_candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_qwen3_embedding", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_qwen3_embedding" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-05", "revision": { "id": "revision_candidate_qwen_20250605_qwen3_embedding_advancing_text_embedding_and_reranki_qwen3_embedding", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-embedding" }, { "id": "source_candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_bd66219782efce3b6337f14c61f546c89e13d37b_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_bd66219782efce3b6337f14c61f546c89e13d37b_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "revision": { "id": "revision_candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_bd66219782efce3b6337f14c61f546c89e13d37b_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/bd66219782efce3b6337f14c61f546c89e13d37b" }, { "id": "source_candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "revision": { "id": "revision_candidate_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-vlo" }, { "id": "source_candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_aeb9b63d20561045bb9359cb3fa786f397708f29_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_aeb9b63d20561045bb9359cb3fa786f397708f29_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-27", "revision": { "id": "revision_candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_aeb9b63d20561045bb9359cb3fa786f397708f29_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/aeb9b63d20561045bb9359cb3fa786f397708f29" }, { "id": "source_candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_qwen_tts", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_qwen_tts" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-27", "revision": { "id": "revision_candidate_qwen_20250627_time_to_speak_some_dialects_qwen_tts_qwen_tts", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-tts" }, { "id": "source_candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_a654383695200a86b635aae18f9ae6ccba7b20da_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_a654383695200a86b635aae18f9ae6ccba7b20da_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "revision": { "id": "revision_candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_a654383695200a86b635aae18f9ae6ccba7b20da_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/a654383695200a86b635aae18f9ae6ccba7b20da" }, { "id": "source_candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "revision": { "id": "revision_candidate_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-coder" }, { "id": "source_candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_d3913d3fad8cd06ee020225844573b31de982e19_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_d3913d3fad8cd06ee020225844573b31de982e19_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-24", "revision": { "id": "revision_candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_d3913d3fad8cd06ee020225844573b31de982e19_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/d3913d3fad8cd06ee020225844573b31de982e19" }, { "id": "source_candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_qwen_mt", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_qwen_mt" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-24", "revision": { "id": "revision_candidate_qwen_20250724_qwen_mt_where_speed_meets_smart_translation_qwen_mt", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-mt" }, { "id": "source_candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_b42c93786a6c8dbf729ec3a41636fda88b04d6e9_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_b42c93786a6c8dbf729ec3a41636fda88b04d6e9_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-27", "revision": { "id": "revision_candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_b42c93786a6c8dbf729ec3a41636fda88b04d6e9_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/b42c93786a6c8dbf729ec3a41636fda88b04d6e9" }, { "id": "source_candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_gspo", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_gspo" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-27", "revision": { "id": "revision_candidate_qwen_20250727_gspo_towards_scalable_reinforcement_learning_for_lan_gspo", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/gspo" }, { "id": "source_candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_51eec55b76cc7760326919ad5ab4987823ca3a4f_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_51eec55b76cc7760326919ad5ab4987823ca3a4f_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-04", "revision": { "id": "revision_candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_51eec55b76cc7760326919ad5ab4987823ca3a4f_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/51eec55b76cc7760326919ad5ab4987823ca3a4f" }, { "id": "source_candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_qwen_image", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_qwen_image" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-04", "revision": { "id": "revision_candidate_qwen_20250804_qwen_image_crafting_with_native_text_rendering_qwen_image", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-image" }, { "id": "source_candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_3658ef85328511103029b904cb0f3fda78187735_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_3658ef85328511103029b904cb0f3fda78187735_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-18", "revision": { "id": "revision_candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_3658ef85328511103029b904cb0f3fda78187735_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/3658ef85328511103029b904cb0f3fda78187735" }, { "id": "source_candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_qwen_image_edit", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_qwen_image_edit" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-18", "revision": { "id": "revision_candidate_qwen_20250818_qwen_image_edit_image_editing_with_higher_quality_an_qwen_image_edit", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-image-edit" }, { "id": "source_candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_41e4c0f6175f9b004a03a07e42343eaaf48329e7_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_41e4c0f6175f9b004a03a07e42343eaaf48329e7_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-08", "revision": { "id": "revision_candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_41e4c0f6175f9b004a03a07e42343eaaf48329e7_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/41e4c0f6175f9b004a03a07e42343eaaf48329e7" }, { "id": "source_candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_qwen3_asr_flash", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_qwen3_asr_flash" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-08", "revision": { "id": "revision_candidate_qwen_20250908_qwen3_asr_hear_clearly_transcribe_smartly_qwen3_asr_flash", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-asr-flash" }, { "id": "source_candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_4074cca80393150c248e508aa62983f9cb7d27cd_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_4074cca80393150c248e508aa62983f9cb7d27cd_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "revision": { "id": "revision_candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_4074cca80393150c248e508aa62983f9cb7d27cd_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/4074cca80393150c248e508aa62983f9cb7d27cd" }, { "id": "source_candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "revision": { "id": "revision_candidate_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-next" }, { "id": "source_candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_65f766fc2dcba7905c1cb69cc4cab90e94126bf4_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_65f766fc2dcba7905c1cb69cc4cab90e94126bf4_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "revision": { "id": "revision_candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_65f766fc2dcba7905c1cb69cc4cab90e94126bf4_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/65f766fc2dcba7905c1cb69cc4cab90e94126bf4" }, { "id": "source_candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "revision": { "id": "revision_candidate_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-omni" }, { "id": "source_candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_b4264e11fb80b5e37350790121baf0a0f10daf82_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_b4264e11fb80b5e37350790121baf0a0f10daf82_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "revision": { "id": "revision_candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_b4264e11fb80b5e37350790121baf0a0f10daf82_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/b4264e11fb80b5e37350790121baf0a0f10daf82" }, { "id": "source_candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_qwen3_tts", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_qwen3_tts" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "revision": { "id": "revision_candidate_qwen_20250921_qwen3_tts_flash_multi_timbre_multi_lingual_multi_dia_qwen3_tts", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-tts" }, { "id": "source_candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_b2de6ae8555599bf3b87eec55a285cdf496b78e4_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_b2de6ae8555599bf3b87eec55a285cdf496b78e4_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_b2de6ae8555599bf3b87eec55a285cdf496b78e4_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/b2de6ae8555599bf3b87eec55a285cdf496b78e4" }, { "id": "source_candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_qwen3_livetranslate", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_qwen3_livetranslate" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_qwen_20250922_qwen3_livetranslate_real_time_multimodal_interpretat_qwen3_livetranslate", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-livetranslate" }, { "id": "source_candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_99f0335c4ad9ff6153e517418d48535ab6d8afef_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_99f0335c4ad9ff6153e517418d48535ab6d8afef_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_99f0335c4ad9ff6153e517418d48535ab6d8afef_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/99f0335c4ad9ff6153e517418d48535ab6d8afef" }, { "id": "source_candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-vl" }, { "id": "source_candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_3e4179176f4f2b028b342fb0193c192046ca291b_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_3e4179176f4f2b028b342fb0193c192046ca291b_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_3e4179176f4f2b028b342fb0193c192046ca291b_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/3e4179176f4f2b028b342fb0193c192046ca291b" }, { "id": "source_candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_qwen3guard", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_qwen3guard" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_qwen_20250922_qwen3guard_real_time_safety_for_your_token_stream_qwen3guard", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3guard" }, { "id": "source_candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_7a90090115ee193ce6a7f619522771dd9696dd93_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_7a90090115ee193ce6a7f619522771dd9696dd93_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_7a90090115ee193ce6a7f619522771dd9696dd93_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/7a90090115ee193ce6a7f619522771dd9696dd93" }, { "id": "source_candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_qwen_image_edit_2509", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_qwen_image_edit_2509" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_qwen_20250922_qwen_image_edit_2509_multi_image_support_improved_co_qwen_image_edit_2509", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-image-edit-2509" }, { "id": "source_candidate_qwen_20250922_travel_planner_your_smart_travel_designer_6228c39fa97ce1a9e65f9e7ce98787a743007fe4_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250922_travel_planner_your_smart_travel_designer_6228c39fa97ce1a9e65f9e7ce98787a743007fe4_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_qwen_20250922_travel_planner_your_smart_travel_designer_6228c39fa97ce1a9e65f9e7ce98787a743007fe4_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/6228c39fa97ce1a9e65f9e7ce98787a743007fe4" }, { "id": "source_candidate_qwen_20250922_travel_planner_your_smart_travel_designer_agent", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250922_travel_planner_your_smart_travel_designer_agent" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_candidate_qwen_20250922_travel_planner_your_smart_travel_designer_agent", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/agent" }, { "id": "source_candidate_qwen_20250924_qwen3_max_just_scale_it_241398b9cd6353de490b0f82806c7848c5d2777d_alias_1", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250924_qwen3_max_just_scale_it_241398b9cd6353de490b0f82806c7848c5d2777d_alias_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "revision": { "id": "revision_candidate_qwen_20250924_qwen3_max_just_scale_it_241398b9cd6353de490b0f82806c7848c5d2777d_alias_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/241398b9cd6353de490b0f82806c7848c5d2777d" }, { "id": "source_candidate_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "revision": { "id": "revision_candidate_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-max" }, { "id": "source_candidate_qwen_20251112_qwen_deepresearch_when_inspiration_becomes_its_own_r_qwen_deepresearch", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20251112_qwen_deepresearch_when_inspiration_becomes_its_own_r_qwen_deepresearch" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "revision": { "id": "revision_candidate_qwen_20251112_qwen_deepresearch_when_inspiration_becomes_its_own_r_qwen_deepresearch", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-deepresearch" }, { "id": "source_candidate_qwen_20251204_qwen3_tts_update_49_timbres_10_languages_9_dialects_qwen3_tts_1128", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20251204_qwen3_tts_update_49_timbres_10_languages_9_dialects_qwen3_tts_1128" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-04", "revision": { "id": "revision_candidate_qwen_20251204_qwen3_tts_update_49_timbres_10_languages_9_dialects_qwen3_tts_1128", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-tts-1128" }, { "id": "source_candidate_qwen_20251204_sapo_a_stable_and_performant_reinforcement_learning_sapo", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20251204_sapo_a_stable_and_performant_reinforcement_learning_sapo" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-04", "revision": { "id": "revision_candidate_qwen_20251204_sapo_a_stable_and_performant_reinforcement_learning_sapo", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/sapo" }, { "id": "source_candidate_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "revision": { "id": "revision_candidate_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-omni-flash-20251201" }, { "id": "source_candidate_qwen_20251219_qwen_image_layered_layered_decomposition_for_inheren_qwen_image_layered", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20251219_qwen_image_layered_layered_decomposition_for_inheren_qwen_image_layered" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-19", "revision": { "id": "revision_candidate_qwen_20251219_qwen_image_layered_layered_decomposition_for_inheren_qwen_image_layered", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-image-layered" }, { "id": "source_candidate_qwen_20251222_qwen3_tts_steps_up_voice_cloning_and_voice_design_qwen3_tts_vc_voicedesign", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20251222_qwen3_tts_steps_up_voice_cloning_and_voice_design_qwen3_tts_vc_voicedesign" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-22", "revision": { "id": "revision_candidate_qwen_20251222_qwen3_tts_steps_up_voice_cloning_and_voice_design_qwen3_tts_vc_voicedesign", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-tts-vc-voicedesign" }, { "id": "source_candidate_qwen_20251223_qwen_image_edit_2511_improve_consistency_qwen_image_edit_2511", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20251223_qwen_image_edit_2511_improve_consistency_qwen_image_edit_2511" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-23", "revision": { "id": "revision_candidate_qwen_20251223_qwen_image_edit_2511_improve_consistency_qwen_image_edit_2511", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-image-edit-2511" }, { "id": "source_candidate_qwen_20251231_qwen_image_2512", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20251231_qwen_image_2512" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-31", "revision": { "id": "revision_candidate_qwen_20251231_qwen_image_2512", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-image-2512" }, { "id": "source_candidate_qwen_20260108_qwen3_vl_embedding", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260108_qwen3_vl_embedding" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-08", "revision": { "id": "revision_candidate_qwen_20260108_qwen3_vl_embedding", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-vl-embedding" }, { "id": "source_candidate_qwen_20260122_qwen3tts_0115", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260122_qwen3tts_0115" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-22", "revision": { "id": "revision_candidate_qwen_20260122_qwen3tts_0115", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3tts-0115" }, { "id": "source_candidate_qwen_20260126_qwen3_max_thinking", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260126_qwen3_max_thinking" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "revision": { "id": "revision_candidate_qwen_20260126_qwen3_max_thinking", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-max-thinking" }, { "id": "source_candidate_qwen_20260129_qwen3asr", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260129_qwen3asr" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-29", "revision": { "id": "revision_candidate_qwen_20260129_qwen3asr", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3asr" }, { "id": "source_candidate_qwen_20260203_qwen3_coder_next", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260203_qwen3_coder_next" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "revision": { "id": "revision_candidate_qwen_20260203_qwen3_coder_next", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-coder-next" }, { "id": "source_candidate_qwen_20260210_qwen_image_2_0", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260210_qwen_image_2_0" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-10", "revision": { "id": "revision_candidate_qwen_20260210_qwen_image_2_0", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-image-2.0" }, { "id": "source_candidate_qwen_20260216_qwen3_5", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260216_qwen3_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "revision": { "id": "revision_candidate_qwen_20260216_qwen3_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.5" }, { "id": "source_candidate_qwen_20260319_qwen3_5_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260319_qwen3_5_max_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-19", "revision": { "id": "revision_candidate_qwen_20260319_qwen3_5_max_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.5-max-preview" }, { "id": "source_candidate_qwen_20260330_qwen3_5_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260330_qwen3_5_omni" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "revision": { "id": "revision_candidate_qwen_20260330_qwen3_5_omni", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.5-omni" }, { "id": "source_candidate_qwen_20260402_qwen3_6", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260402_qwen3_6" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "revision": { "id": "revision_candidate_qwen_20260402_qwen3_6", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.6" }, { "id": "source_candidate_qwen_20260415_qwen3_6_35b_a3b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "revision": { "id": "revision_candidate_qwen_20260415_qwen3_6_35b_a3b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.6-35b-a3b" }, { "id": "source_candidate_qwen_20260418_qwen3_6_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260418_qwen3_6_max_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "revision": { "id": "revision_candidate_qwen_20260418_qwen3_6_max_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.6-max-preview" }, { "id": "source_candidate_qwen_20260422_qwen3_6_27b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260422_qwen3_6_27b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "revision": { "id": "revision_candidate_qwen_20260422_qwen3_6_27b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.6-27b" }, { "id": "source_candidate_qwen_20260428_flashqla", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260428_flashqla" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-28", "revision": { "id": "revision_candidate_qwen_20260428_flashqla", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/flashqla" }, { "id": "source_candidate_qwen_20260430_qwen_scope", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260430_qwen_scope" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-30", "revision": { "id": "revision_candidate_qwen_20260430_qwen_scope", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-scope" }, { "id": "source_candidate_qwen_20260519_qwen3_5_livetranslate", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260519_qwen3_5_livetranslate" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "revision": { "id": "revision_candidate_qwen_20260519_qwen3_5_livetranslate", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.5-livetranslate" }, { "id": "source_candidate_qwen_20260520_qwen3_7", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260520_qwen3_7" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "revision": { "id": "revision_candidate_qwen_20260520_qwen3_7", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.7" }, { "id": "source_candidate_qwen_20260529_qwenvla", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260529_qwenvla" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-29", "revision": { "id": "revision_candidate_qwen_20260529_qwenvla", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwenvla" }, { "id": "source_candidate_qwen_20260601_qwen3_7_plus", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260601_qwen3_7_plus" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "revision": { "id": "revision_candidate_qwen_20260601_qwen3_7_plus", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.7-plus" }, { "id": "source_candidate_qwen_20260616_qwen_robotmanip", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260616_qwen_robotmanip" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-16", "revision": { "id": "revision_candidate_qwen_20260616_qwen_robotmanip", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-robotmanip" }, { "id": "source_candidate_qwen_20260616_qwen_robotnav", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260616_qwen_robotnav" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-16", "revision": { "id": "revision_candidate_qwen_20260616_qwen_robotnav", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-robotnav" }, { "id": "source_candidate_qwen_20260616_qwen_robotsuite", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260616_qwen_robotsuite" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-16", "revision": { "id": "revision_candidate_qwen_20260616_qwen_robotsuite", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-robotsuite" }, { "id": "source_candidate_qwen_20260616_qwen_robotworld", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260616_qwen_robotworld" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-16", "revision": { "id": "revision_candidate_qwen_20260616_qwen_robotworld", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-robotworld" }, { "id": "source_candidate_qwen_20260623_qwen_agentworld", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260623_qwen_agentworld" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-23", "revision": { "id": "revision_candidate_qwen_20260623_qwen_agentworld", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-agentworld" }, { "id": "source_candidate_qwen_20260721_qwen_image_3_0", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260721_qwen_image_3_0" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "revision": { "id": "revision_candidate_qwen_20260721_qwen_image_3_0", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-image-3.0" }, { "id": "source_candidate_qwen_20260803_qwen3_8", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260803_qwen3_8" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "revision": { "id": "revision_candidate_qwen_20260803_qwen3_8", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.8" }, { "id": "source_candidate_qwen_20260826_qwen3_8_flash_next", "lab_id": "qwen", "provenance": { "record_ids": [ "source_candidate_qwen_20260826_qwen3_8_flash_next" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "revision": { "id": "revision_candidate_qwen_20260826_qwen3_8_flash_next", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_official_launch_surface" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.8-flash-next" }, { "id": "source_deepseek_source_hf_r1_0528", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_hf_r1_0528" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "revision": { "id": "revision_deepseek_revision_source_hf_r1_0528", "identifier": "4236a6af538feda4548eca9ab308586007567f52" }, "source_type": "official_hugging_face_card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md" }, { "id": "source_deepseek_source_hf_v25", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_hf_v25" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "revision": { "id": "revision_deepseek_revision_source_hf_v25", "identifier": "c85b5ede86f2a598af339624cac5723861e557ed" }, "source_type": "official_hugging_face_card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V2.5/blob/c85b5ede86f2a598af339624cac5723861e557ed/README.md" }, { "id": "source_deepseek_source_hf_v25_1210", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_hf_v25_1210" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "revision": { "id": "revision_deepseek_revision_source_hf_v25_1210", "identifier": "6f134cbe88cb9284a8ce696e8ac8eefd0bc24ede" }, "source_type": "official_hugging_face_card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V2.5-1210/blob/6f134cbe88cb9284a8ce696e8ac8eefd0bc24ede/README.md" }, { "id": "source_deepseek_source_hf_v31", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_hf_v31" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "revision": { "id": "revision_deepseek_revision_source_hf_v31", "identifier": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db" }, "source_type": "official_hugging_face_card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md" }, { "id": "source_deepseek_source_hf_v31_terminus", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_hf_v31_terminus" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_deepseek_revision_source_hf_v31_terminus", "identifier": "19510d6dc61f79dbd925bd51ee8a9081c509a4b6" }, "source_type": "official_hugging_face_card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md" }, { "id": "source_deepseek_source_hf_v32", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_hf_v32" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "revision": { "id": "revision_deepseek_revision_source_hf_v32", "identifier": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6" }, "source_type": "official_hugging_face_card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md" }, { "id": "source_deepseek_source_hf_v32_exp", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_hf_v32_exp" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "revision": { "id": "revision_deepseek_revision_source_hf_v32_exp", "identifier": "194c67e12b1b0d6df0ef373ddcf215bc84027409" }, "source_type": "official_hugging_face_card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp/blob/194c67e12b1b0d6df0ef373ddcf215bc84027409/README.md" }, { "id": "source_deepseek_source_hf_v32_speciale", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_hf_v32_speciale" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "revision": { "id": "revision_deepseek_revision_source_hf_v32_speciale", "identifier": "c562883eda916fd699a44bbf1f4987a53b96008b" }, "source_type": "official_hugging_face_card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Speciale/blob/c562883eda916fd699a44bbf1f4987a53b96008b/README.md" }, { "id": "source_deepseek_source_hf_v3_0324", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_hf_v3_0324" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "revision": { "id": "revision_deepseek_revision_source_hf_v3_0324", "identifier": "e9b33add76883f293d6bf61f6bd89b497e80e335" }, "source_type": "official_hugging_face_card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324/blob/e9b33add76883f293d6bf61f6bd89b497e80e335/README.md" }, { "id": "source_deepseek_source_hf_v4", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_hf_v4" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "revision": { "id": "revision_deepseek_revision_source_hf_v4", "identifier": "b5968e9190ef611bbf34a7229255be88a0e937c1" }, "source_type": "official_hugging_face_card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/b5968e9190ef611bbf34a7229255be88a0e937c1/README.md" }, { "id": "source_deepseek_source_launch_deepseek_20240517_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20240517_deepseek_chat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-17", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20240517_deepseek_chat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/updates/" }, { "id": "source_deepseek_source_launch_deepseek_20240614_deepseek_coder", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20240614_deepseek_coder" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-14", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20240614_deepseek_coder", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/updates/" }, { "id": "source_deepseek_source_launch_deepseek_20240628_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20240628_deepseek_chat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-28", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20240628_deepseek_chat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/updates/" }, { "id": "source_deepseek_source_launch_deepseek_20240724_deepseek_coder", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20240724_deepseek_coder" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-24", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20240724_deepseek_coder", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/updates/" }, { "id": "source_deepseek_source_launch_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-05", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20240905_deepseek_coder_deepseek_chat_upgraded_to_deepseek_v2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news0905/" }, { "id": "source_deepseek_source_launch_deepseek_20241120_deepseek_r1_lite_preview", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20241120_deepseek_r1_lite_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-20", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20241120_deepseek_r1_lite_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news1120/" }, { "id": "source_deepseek_source_launch_deepseek_20241210_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20241210_deepseek_chat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-10", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20241210_deepseek_chat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news1210/" }, { "id": "source_deepseek_source_launch_deepseek_20241226_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20241226_deepseek_chat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20241226_deepseek_chat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news1226/" }, { "id": "source_deepseek_source_launch_deepseek_20250120_deepseek_reasoner", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20250120_deepseek_reasoner" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20250120_deepseek_reasoner", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250120/" }, { "id": "source_deepseek_source_launch_deepseek_20250324_deepseek_chat", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20250324_deepseek_chat" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-24", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20250324_deepseek_chat", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250325/" }, { "id": "source_deepseek_source_launch_deepseek_20250528_deepseek_reasoner", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20250528_deepseek_reasoner" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-28", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20250528_deepseek_reasoner", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250528/" }, { "id": "source_deepseek_source_launch_deepseek_20250821_deepseek_v3_1", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20250821_deepseek_v3_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-21", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20250821_deepseek_v3_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250821/" }, { "id": "source_deepseek_source_launch_deepseek_20250922_deepseek_v3_1_terminus", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20250922_deepseek_v3_1_terminus" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20250922_deepseek_v3_1_terminus", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250922/" }, { "id": "source_deepseek_source_launch_deepseek_20250929_deepseek_v3_2_exp", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20250929_deepseek_v3_2_exp" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20250929_deepseek_v3_2_exp", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news250929/" }, { "id": "source_deepseek_source_launch_deepseek_20251201_deepseek_v3_2", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20251201_deepseek_v3_2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20251201_deepseek_v3_2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news251201/" }, { "id": "source_deepseek_source_launch_deepseek_20251201_deepseek_v3_2_speciale", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20251201_deepseek_v3_2_speciale" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20251201_deepseek_v3_2_speciale", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news251201/" }, { "id": "source_deepseek_source_launch_deepseek_20260424_deepseek_v4", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20260424_deepseek_v4" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-24", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20260424_deepseek_v4", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news260424/" }, { "id": "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20260731_deepseek_v4_flash_update" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-31", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20260731_deepseek_v4_flash_update", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/updates/" }, { "id": "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20260813_deepseek_v4_pro_update" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20260813_deepseek_v4_pro_update", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news260813/" }, { "id": "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-21", "revision": { "id": "revision_deepseek_revision_source_launch_deepseek_20260821_deepseek_v4_flash_vision_exp_release", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://api-docs.deepseek.com/news/news260821/" }, { "id": "source_deepseek_source_repo_v3", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_repo_v3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-26", "revision": { "id": "revision_deepseek_revision_source_repo_v3", "identifier": "9b4e9788e4a3a731f7567338ed15d3ec549ce03b" }, "source_type": "official_repository", "url": "https://github.com/deepseek-ai/DeepSeek-V3/blob/9b4e9788e4a3a731f7567338ed15d3ec549ce03b/README.md" }, { "id": "source_deepseek_source_report_r1", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_report_r1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-20", "revision": { "id": "revision_deepseek_revision_source_report_r1", "identifier": "0cf78561f1d51c84a21b2190626b21116d5c68bb" }, "source_type": "technical_report", "url": "https://github.com/deepseek-ai/DeepSeek-R1/blob/0cf78561f1d51c84a21b2190626b21116d5c68bb/DeepSeek_R1.pdf" }, { "id": "source_deepseek_source_report_v32", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_report_v32" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-01", "revision": { "id": "revision_deepseek_revision_source_report_v32", "identifier": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6" }, "source_type": "technical_report", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/assets/paper.pdf" }, { "id": "source_deepseek_source_report_v32_exp", "lab_id": "deepseek", "provenance": { "record_ids": [ "source_deepseek_source_report_v32_exp" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-29", "revision": { "id": "revision_deepseek_revision_source_report_v32_exp", "identifier": "87e509a2e5a100d221c97df52c6e8be7835f0057" }, "source_type": "technical_report", "url": "https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/blob/87e509a2e5a100d221c97df52c6e8be7835f0057/DeepSeek_V3_2.pdf" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20240208_the_next_chapter_of_our_gemini_era" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-08", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20240208_the_next_chapter_of_our_gemini_era", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/ai/google-gemini-update-sundar-pichai-2024" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20240215_our_next_generation_model_gemini_1_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-15", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20240215_our_next_generation_model_gemini_1_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/products/google-gemini-next-generation-model-february-2024" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20240221_gemma_introducing_new_state_of_the_art_open_models", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/gemma-open-models" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20240514_gemini_breaks_new_ground_a_faster_model_longer_conte", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/products/google-gemini-update-flash-ai-assistant-io-2024" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-24", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20240924_updated_production_ready_gemini_models_reduced_1_5_p", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/updated-gemini-models-reduced-15-pro-pricing-increased-rate-limits-and-more" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20241211_introducing_gemini_2_0_our_new_ai_model_for_the_agen", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/google-gemini-ai-update-december-2024" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-05", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250205_gemini_2_0_is_now_available_to_everyone", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-updates-february-2025" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-25", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250225_start_building_with_gemini_2_0_flash_and_flash_lite", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/start-building-with-the-gemini-2-0-flash-family" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250312_experiment_with_gemini_2_0_flash_native_image_genera", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/experiment-with-gemini-20-flash-native-image-generation" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250312_introducing_gemma_3", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250312_introducing_gemma_3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-12", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250312_introducing_gemma_3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/gemma-3" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250325_gemini_2_5_our_most_intelligent_ai_model", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250417_introducing_gemini_2_5_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250417_introducing_gemini_2_5_flash" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250417_introducing_gemini_2_5_flash", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/start-building-with-gemini-25-flash" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250506_build_rich_interactive_web_apps_with_an_updated_gemi", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/gemini-2-5-pro-updates" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-06", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250506_gemini_2_5_pro_preview_even_better_coding_performanc", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/gemini-2-5-pro-io-improved-coding-performance" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250520_announcing_gemma_3n_preview_powerful_efficient_mobil", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/introducing-gemma-3n" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250520_gemini_2_5_our_most_intelligent_models_are_getting_e", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/google-gemini-updates-io-2025" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-17", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250617_gemini_2_5_updates_to_our_family_of_thinking_models", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/gemini-2-5-thinking-model-updates" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250626_introducing_gemma_3n_the_developer_guide", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/introducing-gemma-3n-developer-guide" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250709_medgemma_our_most_capable_open_models_for_health_ai", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://research.google/blog/medgemma-our-most-capable-open-models-for-health-ai-development" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-09", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250709_t5gemma_a_new_collection_of_encoder_decoder_gemma_mo", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/t5gemma" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250722_gemini_2_5_flash_lite_is_now_ready_for_scaled_produc", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/gemini-25-flash-lite-is-now-stable-and-generally-available" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250801_try_deep_think_in_the_gemini_app" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250801_try_deep_think_in_the_gemini_app", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/gemini-2-5-deep-think" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-14", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250814_introducing_gemma_3_270m_the_compact_model_for_hyper", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://developers.googleblog.com/en/introducing-gemma-3-270m" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20250912_vaultgemma_the_world_s_most_capable_differentially_p", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://research.google/blog/vaultgemma-the-worlds-most-capable-differentially-private-llm" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20251007_introducing_the_gemini_2_5_computer_use_model", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-computer-use-model" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20251118_a_new_era_of_intelligence_with_gemini_3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-17", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20251217_gemini_3_flash_frontier_intelligence_built_for_speed", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3-flash" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-11", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260211_accelerating_mathematical_and_scientific_discovery_w", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/accelerating-mathematical-and-scientific-discovery-with-gemini-deep-think" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260212_gemini_3_deep_think_advancing_science_research_and_e", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-deep-think" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-19", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260219_gemini_3_1_pro_a_smarter_model_for_your_most_complex", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260303_gemini_3_1_flash_lite_built_for_intelligence_at_scal", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-flash-lite" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260402_gemma_4_byte_for_byte_the_most_capable_open_models", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-19", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260519_gemini_3_5_frontier_intelligence_with_action", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260603_introducing_gemma_4_12b_a_unified_encoder_free_multi", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/introducing-gemma-4-12b" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-10", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260610_diffusiongemma_numeric_speed_claim_redacted_faster_text_generation", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/diffusion-gemma-faster-text-generation" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-24", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260624_introducing_computer_use_in_gemini_3_5_flash", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-computer-use-gemini-3-5-flash" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_5_flash_cyber", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://deepmind.google/blog/introducing-gemini-3-5-flash-cyber" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-21", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260721_introducing_gemini_3_6_flash_3_5_flash_lite_and_3_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-6-flash-3-5-flash-lite-3-5-flash-cyber" }, { "id": "source_google_deepmind_source_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_source_launch_google_deepmind_20260813_introducing_gemini_3_7_flash" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "revision": { "id": "revision_google_deepmind_revision_launch_google_deepmind_20260813_introducing_gemini_3_7_flash", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-3-7-flash" }, { "id": "source_google_deepmind_support_computer_use_eval_info", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_computer_use_eval_info" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "revision": { "id": "revision_google_deepmind_revision_support_computer_use_eval_info", "retrieval_date": "2026-09-01", "unavailable_reason": "no_source_issued_revision_identifier" }, "source_type": "technical_report", "url": "https://storage.googleapis.com/deepmind-media/gemini/computer_use_eval_additional_info.pdf" }, { "id": "source_google_deepmind_support_computer_use_system_card", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_computer_use_system_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-10-07", "revision": { "id": "revision_google_deepmind_revision_support_computer_use_system_card", "identifier": "model_card_published_2025_10_07" }, "source_type": "system_card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Computer-Use-Model-Card.pdf" }, { "id": "source_google_deepmind_support_gemini_1_5_report", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_gemini_1_5_report" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-14", "revision": { "id": "revision_google_deepmind_revision_support_gemini_1_5_report", "retrieval_date": "2026-09-01", "unavailable_reason": "no_source_issued_revision_identifier" }, "source_type": "technical_report", "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf" }, { "id": "source_google_deepmind_support_gemini_2_5_deep_think_card", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_gemini_2_5_deep_think_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-01", "revision": { "id": "revision_google_deepmind_revision_support_gemini_2_5_deep_think_card", "identifier": "model_card_published_updated_2025_08_01" }, "source_type": "model_card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Deep-Think-Model-Card.pdf" }, { "id": "source_google_deepmind_support_gemini_2_5_flash_card", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_gemini_2_5_flash_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "revision": { "id": "revision_google_deepmind_revision_support_gemini_2_5_flash_card", "retrieval_date": "2026-09-01", "unavailable_reason": "linked_source_unavailable_at_retrieval" }, "source_type": "model_card", "url": "https://storage.googleapis.com/model-cards/documents/gemini-2.5-flash-preview.pdf" }, { "id": "source_google_deepmind_support_gemini_2_5_pro_card", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_gemini_2_5_pro_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-20", "revision": { "id": "revision_google_deepmind_revision_support_gemini_2_5_pro_card", "retrieval_date": "2026-09-01", "unavailable_reason": "linked_source_unavailable_at_retrieval" }, "source_type": "model_card", "url": "https://storage.googleapis.com/model-cards/documents/gemini-2.5-pro-preview.pdf" }, { "id": "source_google_deepmind_support_gemini_3_7_flash_model_card", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_gemini_3_7_flash_model_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-13", "revision": { "id": "revision_google_deepmind_revision_support_gemini_3_7_flash_model_card", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_route_content_mismatch" }, "source_type": "model_card", "url": "https://deepmind.google/models/model-cards/gemini-3-7-flash" }, { "id": "source_google_deepmind_support_gemini_3_eval_methodology", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_gemini_3_eval_methodology" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "revision": { "id": "revision_google_deepmind_revision_support_gemini_3_eval_methodology", "retrieval_date": "2026-09-01", "unavailable_reason": "client_rendered_content_not_exposed_in_readback" }, "source_type": "technical_report", "url": "https://deepmind.google/models/evals-methodology/gemini-3-pro" }, { "id": "source_google_deepmind_support_gemini_3_model_card", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_gemini_3_model_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-18", "revision": { "id": "revision_google_deepmind_revision_support_gemini_3_model_card", "retrieval_date": "2026-09-01", "unavailable_reason": "client_rendered_content_not_exposed_in_readback" }, "source_type": "model_card", "url": "https://deepmind.google/models/model-cards/gemini-3-pro" }, { "id": "source_google_deepmind_support_gemma_3_report", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_gemma_3_report" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-25", "revision": { "id": "revision_google_deepmind_revision_support_gemma_3_report", "identifier": "arxiv_2503_19786v1" }, "source_type": "technical_report", "url": "https://storage.googleapis.com/deepmind-media/gemma/Gemma3Report.pdf" }, { "id": "source_google_deepmind_support_gemma_4_model_card", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_gemma_4_model_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "revision": { "id": "revision_google_deepmind_revision_support_gemma_4_model_card", "retrieval_date": "2026-09-01", "unavailable_reason": "no_source_issued_revision_identifier" }, "source_type": "model_card", "url": "https://ai.google.dev/gemma/docs/core/model_card_4" }, { "id": "source_google_deepmind_support_gemma_report", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_gemma_report" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-21", "revision": { "id": "revision_google_deepmind_revision_support_gemma_report", "identifier": "report_date_2024_02_21" }, "source_type": "technical_report", "url": "https://storage.googleapis.com/deepmind-media/gemma/gemma-report.pdf" }, { "id": "source_google_deepmind_support_vaultgemma_report", "lab_id": "google_deepmind", "provenance": { "record_ids": [ "source_google_deepmind_support_vaultgemma_report" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-12", "revision": { "id": "revision_google_deepmind_revision_support_vaultgemma_report", "identifier": "report_date_2025_09_12" }, "source_type": "technical_report", "url": "https://services.google.com/fh/files/blogs/vaultgemma_tech_report.pdf" }, { "id": "source_meta_launch_fair_december", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_fair_december" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-12", "revision": { "id": "revision_meta_revision_launch_fair_december", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-fair-updates-agents-robustness-safety-architecture/" }, { "id": "source_meta_launch_fair_june", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_fair_june" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-18", "revision": { "id": "revision_meta_revision_launch_fair_june", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-fair-research-new-releases/" }, { "id": "source_meta_launch_fair_october", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_fair_october" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-18", "revision": { "id": "revision_meta_revision_launch_fair_october", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/fair-news-segment-anything-2-1-meta-spirit-lm-layer-skip-salsa-lingua/" }, { "id": "source_meta_launch_llama_3", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_llama_3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-18", "revision": { "id": "revision_meta_revision_launch_llama_3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-llama-3/" }, { "id": "source_meta_launch_llama_3_1", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_llama_3_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-23", "revision": { "id": "revision_meta_revision_launch_llama_3_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-llama-3-1/" }, { "id": "source_meta_launch_llama_3_2", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_llama_3_2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-25", "revision": { "id": "revision_meta_revision_launch_llama_3_2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/llama-3-2-connect-2024-vision-edge-mobile-devices/" }, { "id": "source_meta_launch_llama_3_3", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_llama_3_3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-19", "revision": { "id": "revision_meta_revision_launch_llama_3_3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/future-of-ai-built-with-llama/" }, { "id": "source_meta_launch_llama_4", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_llama_4" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "revision": { "id": "revision_meta_revision_launch_llama_4", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/" }, { "id": "source_meta_launch_llama_quantized", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_llama_quantized" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "revision": { "id": "revision_meta_revision_launch_llama_quantized", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-llama-quantized-lightweight-models/" }, { "id": "source_meta_launch_llamacon", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_llamacon" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-29", "revision": { "id": "revision_meta_revision_launch_llamacon", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/llamacon-llama-news/" }, { "id": "source_meta_launch_muse_spark", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_muse_spark" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "revision": { "id": "revision_meta_revision_launch_muse_spark", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/introducing-muse-spark-msl/" }, { "id": "source_meta_launch_muse_spark_1_1", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_muse_spark_1_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "revision": { "id": "revision_meta_revision_launch_muse_spark_1_1", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/" }, { "id": "source_meta_launch_plm", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_launch_plm" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "revision": { "id": "revision_meta_revision_launch_plm", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://ai.meta.com/blog/meta-fair-updates-perception-localization-reasoning/" }, { "id": "source_meta_llama_downloads", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_llama_downloads" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "revision": { "id": "revision_meta_revision_llama_downloads", "retrieval_date": "2026-09-01", "unavailable_reason": "document_exposes_no_revision_identifier" }, "source_type": "launch_page", "url": "https://www.llama.com/llama-downloads/" }, { "id": "source_meta_paper_blt", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_paper_blt" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-13", "revision": { "id": "revision_meta_revision_paper_blt", "identifier": "arxiv_2412_09871v1" }, "source_type": "technical_report", "url": "https://arxiv.org/abs/2412.09871v1" }, { "id": "source_meta_paper_chameleon", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_paper_chameleon" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-16", "revision": { "id": "revision_meta_revision_paper_chameleon", "identifier": "arxiv_2405_09818v2" }, "source_type": "technical_report", "url": "https://arxiv.org/abs/2405.09818v2" }, { "id": "source_meta_paper_layerskip", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_paper_layerskip" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "revision": { "id": "revision_meta_revision_paper_layerskip", "identifier": "arxiv_2404_16710v4" }, "source_type": "technical_report", "url": "https://arxiv.org/abs/2404.16710v4" }, { "id": "source_meta_paper_lcm", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_paper_lcm" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-11", "revision": { "id": "revision_meta_revision_paper_lcm", "identifier": "arxiv_2412_08821v2" }, "source_type": "technical_report", "url": "https://arxiv.org/abs/2412.08821v2" }, { "id": "source_meta_paper_llama_3_herd", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_paper_llama_3_herd" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-31", "revision": { "id": "revision_meta_revision_paper_llama_3_herd", "identifier": "arxiv_2407_21783v3" }, "source_type": "technical_report", "url": "https://arxiv.org/abs/2407.21783v3" }, { "id": "source_meta_paper_mtp", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_paper_mtp" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-30", "revision": { "id": "revision_meta_revision_paper_mtp", "identifier": "arxiv_2404_19737v1" }, "source_type": "technical_report", "url": "https://arxiv.org/abs/2404.19737v1" }, { "id": "source_meta_paper_plm", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_paper_plm" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-17", "revision": { "id": "revision_meta_revision_paper_plm", "identifier": "arxiv_2504_13180v3" }, "source_type": "technical_report", "url": "https://arxiv.org/abs/2504.13180v3" }, { "id": "source_meta_repo_layerskip", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_repo_layerskip" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-03", "revision": { "id": "revision_meta_revision_repo_layerskip", "identifier": "f42c66965545747d42e27d4beab86b5c17956566" }, "source_type": "official_repository", "url": "https://github.com/facebookresearch/LayerSkip/blob/f42c66965545747d42e27d4beab86b5c17956566/README.md" }, { "id": "source_meta_report_muse_1_1", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_report_muse_1_1" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "revision": { "id": "revision_meta_revision_report_muse_1_1", "retrieval_date": "2026-09-01", "unavailable_reason": "document_exposes_no_revision_identifier" }, "source_type": "system_card", "url": "https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report" }, { "id": "source_meta_report_muse_1_1_methodology", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_report_muse_1_1_methodology" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "revision": { "id": "revision_meta_revision_report_muse_1_1_methodology", "retrieval_date": "2026-09-01", "unavailable_reason": "linked_resource_unavailable" }, "source_type": "technical_report", "url": "https://ai.meta.com/static-resource/muse-spark-1-1-eval-methodology" }, { "id": "source_meta_report_muse_methodology", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_report_muse_methodology" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-08", "revision": { "id": "revision_meta_revision_report_muse_methodology", "retrieval_date": "2026-09-01", "unavailable_reason": "document_exposes_no_revision_identifier" }, "source_type": "technical_report", "url": "https://ai.meta.com/static-resource/muse-spark-eval-methodology" }, { "id": "source_meta_report_muse_safety", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_report_muse_safety" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-26", "revision": { "id": "revision_meta_revision_report_muse_safety", "retrieval_date": "2026-09-01", "unavailable_reason": "document_exposes_no_revision_identifier" }, "source_type": "system_card", "url": "https://ai.meta.com/static-resource/muse-spark-safety-and-preparedness-report/" }, { "id": "source_meta_tech_llama_3_1_card", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_tech_llama_3_1_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "revision": { "id": "revision_meta_revision_tech_llama_3_1_card", "identifier": "e2e67cd8b0feca839666b7082dcc1c4fc0ae11e2" }, "source_type": "model_card", "url": "https://github.com/meta-llama/llama-models/blob/e2e67cd8b0feca839666b7082dcc1c4fc0ae11e2/models/llama3_1/MODEL_CARD.md" }, { "id": "source_meta_tech_llama_3_1_eval", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_tech_llama_3_1_eval" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "revision": { "id": "revision_meta_revision_tech_llama_3_1_eval", "identifier": "dd320f7755d582e38f228f97e3ef655765a88b22" }, "source_type": "official_repository", "url": "https://github.com/meta-llama/llama-models/blob/dd320f7755d582e38f228f97e3ef655765a88b22/models/llama3_1/eval_details.md" }, { "id": "source_meta_tech_llama_3_2_eval", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_tech_llama_3_2_eval" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-02", "revision": { "id": "revision_meta_revision_tech_llama_3_2_eval", "identifier": "364ed4ec2bdcc13ab3acb08e364e843b6322b9ae" }, "source_type": "official_repository", "url": "https://github.com/meta-llama/llama-models/blob/364ed4ec2bdcc13ab3acb08e364e843b6322b9ae/models/llama3_2/eval_details.md" }, { "id": "source_meta_tech_llama_3_2_text_card", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_tech_llama_3_2_text_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-10-24", "revision": { "id": "revision_meta_revision_tech_llama_3_2_text_card", "identifier": "4a9ff6aede5cb4e036542132bb4f8b77a3134908" }, "source_type": "model_card", "url": "https://github.com/meta-llama/llama-models/blob/4a9ff6aede5cb4e036542132bb4f8b77a3134908/models/llama3_2/MODEL_CARD.md" }, { "id": "source_meta_tech_llama_3_2_vision_card", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_tech_llama_3_2_vision_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-30", "revision": { "id": "revision_meta_revision_tech_llama_3_2_vision_card", "identifier": "e2e67cd8b0feca839666b7082dcc1c4fc0ae11e2" }, "source_type": "model_card", "url": "https://github.com/meta-llama/llama-models/blob/e2e67cd8b0feca839666b7082dcc1c4fc0ae11e2/models/llama3_2/MODEL_CARD_VISION.md" }, { "id": "source_meta_tech_llama_3_card", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_tech_llama_3_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-11", "revision": { "id": "revision_meta_revision_tech_llama_3_card", "identifier": "e14519ce6a4296858940c68a37bded0a929481d1" }, "source_type": "model_card", "url": "https://github.com/meta-llama/llama3/blob/e14519ce6a4296858940c68a37bded0a929481d1/MODEL_CARD.md" }, { "id": "source_meta_tech_llama_3_eval", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_tech_llama_3_eval" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-26", "revision": { "id": "revision_meta_revision_tech_llama_3_eval", "identifier": "c7c6c2ef0c0109a281db65dd6ecbc96f9f9fa2fe" }, "source_type": "official_repository", "url": "https://github.com/meta-llama/llama3/blob/c7c6c2ef0c0109a281db65dd6ecbc96f9f9fa2fe/eval_details.md" }, { "id": "source_meta_tech_llama_4_card", "lab_id": "meta", "provenance": { "record_ids": [ "source_meta_tech_llama_4_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-05", "revision": { "id": "revision_meta_revision_tech_llama_4_card", "identifier": "038acb9fe1e1ddc5cf1b3989fb07b311a0ffcae4" }, "source_type": "model_card", "url": "https://github.com/meta-llama/llama-models/blob/038acb9fe1e1ddc5cf1b3989fb07b311a0ffcae4/models/llama4/MODEL_CARD.md" }, { "id": "source_openai_source_https_deploymentsafety_openai_com_gpt_5_5", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_https_deploymentsafety_openai_com_gpt_5_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "revision": { "id": "revision_openai_revision_https_deploymentsafety_openai_com_gpt_5_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "system_card", "url": "https://deploymentsafety.openai.com/gpt-5-5" }, { "id": "source_openai_source_https_openai_com_index_openai_o1_system_card", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_https_openai_com_index_openai_o1_system_card" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-05", "revision": { "id": "revision_openai_revision_https_openai_com_index_openai_o1_system_card", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "system_card", "url": "https://openai.com/index/openai-o1-system-card/" }, { "id": "source_openai_source_openai_20240125_new_embedding_models_and_api_updates", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20240125_new_embedding_models_and_api_updates" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "revision": { "id": "revision_openai_revision_openai_20240125_new_embedding_models_and_api_updates", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/new-embedding-models-and-api-updates/" }, { "id": "source_openai_source_openai_20240513_hello_gpt_4o", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20240513_hello_gpt_4o" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-13", "revision": { "id": "revision_openai_revision_openai_20240513_hello_gpt_4o", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/hello-gpt-4o/" }, { "id": "source_openai_source_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-07-18", "revision": { "id": "revision_openai_revision_openai_20240718_gpt_4o_mini_advancing_cost_efficient_intelligence", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/" }, { "id": "source_openai_source_openai_20240912_introducing_openai_o1_preview", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20240912_introducing_openai_o1_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "revision": { "id": "revision_openai_revision_openai_20240912_introducing_openai_o1_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-openai-o1-preview/" }, { "id": "source_openai_source_openai_20240912_openai_o1_mini", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20240912_openai_o1_mini" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-12", "revision": { "id": "revision_openai_revision_openai_20240912_openai_o1_mini", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/openai-o1-mini-advancing-cost-efficient-reasoning/" }, { "id": "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20241217_openai_o1_and_new_tools_for_developers" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-17", "revision": { "id": "revision_openai_revision_openai_20241217_openai_o1_and_new_tools_for_developers", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/o1-and-new-tools-for-developers/" }, { "id": "source_openai_source_openai_20250123_computer_using_agent", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20250123_computer_using_agent" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-23", "revision": { "id": "revision_openai_revision_openai_20250123_computer_using_agent", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/computer-using-agent/" }, { "id": "source_openai_source_openai_20250131_openai_o3_mini", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20250131_openai_o3_mini" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-31", "revision": { "id": "revision_openai_revision_openai_20250131_openai_o3_mini", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/openai-o3-mini/" }, { "id": "source_openai_source_openai_20250227_introducing_gpt_4_5", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20250227_introducing_gpt_4_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-27", "revision": { "id": "revision_openai_revision_openai_20250227_introducing_gpt_4_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-4-5/" }, { "id": "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20250414_introducing_gpt_4_1_in_the_api" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-14", "revision": { "id": "revision_openai_revision_openai_20250414_introducing_gpt_4_1_in_the_api", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-4-1/" }, { "id": "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20250416_introducing_openai_o3_and_o4_mini" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-16", "revision": { "id": "revision_openai_revision_openai_20250416_introducing_openai_o3_and_o4_mini", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-o3-and-o4-mini/" }, { "id": "source_openai_source_openai_20250516_introducing_codex", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20250516_introducing_codex" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-05-16", "revision": { "id": "revision_openai_revision_openai_20250516_introducing_codex", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-codex/" }, { "id": "source_openai_source_openai_20250805_introducing_gpt_oss", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20250805_introducing_gpt_oss" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-05", "revision": { "id": "revision_openai_revision_openai_20250805_introducing_gpt_oss", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-oss/" }, { "id": "source_openai_source_openai_20250807_introducing_gpt_5", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20250807_introducing_gpt_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-08-07", "revision": { "id": "revision_openai_revision_openai_20250807_introducing_gpt_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5/" }, { "id": "source_openai_source_openai_20250915_introducing_upgrades_to_codex", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20250915_introducing_upgrades_to_codex" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-15", "revision": { "id": "revision_openai_revision_openai_20250915_introducing_upgrades_to_codex", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-upgrades-to-codex/" }, { "id": "source_openai_source_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-12", "revision": { "id": "revision_openai_revision_openai_20251112_gpt_5_1_a_smarter_more_conversational_chatgpt", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-1/" }, { "id": "source_openai_source_openai_20251119_building_more_with_gpt_5_1_codex_max", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20251119_building_more_with_gpt_5_1_codex_max" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-11-19", "revision": { "id": "revision_openai_revision_openai_20251119_building_more_with_gpt_5_1_codex_max", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-1-codex-max/" }, { "id": "source_openai_source_openai_20251211_introducing_gpt_5_2", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20251211_introducing_gpt_5_2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-11", "revision": { "id": "revision_openai_revision_openai_20251211_introducing_gpt_5_2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-2/" }, { "id": "source_openai_source_openai_20251218_introducing_gpt_5_2_codex", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20251218_introducing_gpt_5_2_codex" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-18", "revision": { "id": "revision_openai_revision_openai_20251218_introducing_gpt_5_2_codex", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-2-codex/" }, { "id": "source_openai_source_openai_20260205_introducing_gpt_5_3_codex", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260205_introducing_gpt_5_3_codex" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-05", "revision": { "id": "revision_openai_revision_openai_20260205_introducing_gpt_5_3_codex", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-3-codex/" }, { "id": "source_openai_source_openai_20260212_introducing_gpt_5_3_codex_spark", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260212_introducing_gpt_5_3_codex_spark" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-12", "revision": { "id": "revision_openai_revision_openai_20260212_introducing_gpt_5_3_codex_spark", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-3-codex-spark/" }, { "id": "source_openai_source_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-03", "revision": { "id": "revision_openai_revision_openai_20260303_gpt_5_3_instant_smoother_more_useful_everyday_conver", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-3-instant/" }, { "id": "source_openai_source_openai_20260305_introducing_gpt_5_4", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260305_introducing_gpt_5_4" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-05", "revision": { "id": "revision_openai_revision_openai_20260305_introducing_gpt_5_4", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-4/" }, { "id": "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260317_introducing_gpt_5_4_mini_and_nano" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-17", "revision": { "id": "revision_openai_revision_openai_20260317_introducing_gpt_5_4_mini_and_nano", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/" }, { "id": "source_openai_source_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-16", "revision": { "id": "revision_openai_revision_openai_20260416_introducing_gpt_rosalind_for_life_sciences_research", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-rosalind/" }, { "id": "source_openai_source_openai_20260423_introducing_gpt_5_5", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260423_introducing_gpt_5_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-23", "revision": { "id": "revision_openai_revision_openai_20260423_introducing_gpt_5_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-gpt-5-5/" }, { "id": "source_openai_source_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-05", "revision": { "id": "revision_openai_revision_openai_20260505_gpt_5_5_instant_smarter_clearer_and_more_personalize", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-5-instant/" }, { "id": "source_openai_source_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260603_introducing_new_capabilities_to_gpt_rosalind" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-03", "revision": { "id": "revision_openai_revision_openai_20260603_introducing_new_capabilities_to_gpt_rosalind", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/introducing-new-capabilities-to-gpt-rosalind/" }, { "id": "source_openai_source_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-26", "revision": { "id": "revision_openai_revision_openai_20260626_previewing_gpt_5_6_sol_a_next_generation_model", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/previewing-gpt-5-6-sol/" }, { "id": "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-09", "revision": { "id": "revision_openai_revision_openai_20260709_gpt_5_6_frontier_intelligence_that_scales_with_your", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/gpt-5-6/" }, { "id": "source_openai_source_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-07-30", "revision": { "id": "revision_openai_revision_openai_20260730_advancing_the_price_performance_frontier_with_gpt_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/advancing-the-price-performance-frontier-with-gpt-5-6/" }, { "id": "source_openai_source_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-06", "revision": { "id": "revision_openai_revision_openai_20260806_improving_gpt_5_6_sol_in_chatgpt_and_expanding", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/improving-gpt-5-6-sol-in-chatgpt/" }, { "id": "source_openai_source_openai_20260811_daybreak_models_are_now_available_on_aws", "lab_id": "openai", "provenance": { "record_ids": [ "source_openai_source_openai_20260811_daybreak_models_are_now_available_on_aws" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-11", "revision": { "id": "revision_openai_revision_openai_20260811_daybreak_models_are_now_available_on_aws", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page" }, "source_type": "launch_page", "url": "https://openai.com/index/daybreak-models-are-now-available-on-aws/" }, { "id": "source_qwen_source_qwen_20240123_introducing_qwen_qwen", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240123_introducing_qwen_qwen" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-23", "revision": { "id": "revision_qwen_revision_qwen_20240123_introducing_qwen_qwen", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen" }, { "id": "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240125_introducing_qwen_vl_qwen_vl" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-01-25", "revision": { "id": "revision_qwen_revision_qwen_20240125_introducing_qwen_vl_qwen_vl", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-vl" }, { "id": "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240204_introducing_qwen1_5_qwen1_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-02-04", "revision": { "id": "revision_qwen_revision_qwen_20240204_introducing_qwen1_5_qwen1_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen1.5" }, { "id": "source_qwen_source_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-03-28", "revision": { "id": "revision_qwen_revision_qwen_20240328_qwen1_5_moe_matching_7b_model_performance_with_1_3_a_qwen_moe", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-moe" }, { "id": "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-02", "revision": { "id": "revision_qwen_revision_qwen_20240402_qwen1_5_32b_fitting_the_capstone_of_the_qwen1_5_lang_qwen1_5_32b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen1.5-32b" }, { "id": "source_qwen_source_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-16", "revision": { "id": "revision_qwen_revision_qwen_20240416_code_with_codeqwen1_5_codeqwen1_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/codeqwen1.5" }, { "id": "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-04-25", "revision": { "id": "revision_qwen_revision_qwen_20240425_qwen1_5_110b_the_first_100b_model_of_the_qwen1_5_ser_qwen1_5_110b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen1.5-110b" }, { "id": "source_qwen_source_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-05-11", "revision": { "id": "revision_qwen_revision_qwen_20240511_notes_on_qwen_max_0428_qwen_max_0428", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-max-0428" }, { "id": "source_qwen_source_qwen_20240606_hello_qwen2_qwen2", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240606_hello_qwen2_qwen2" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-06-06", "revision": { "id": "revision_qwen_revision_qwen_20240606_hello_qwen2_qwen2", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2" }, { "id": "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240807_introducing_qwen2_math_qwen2_math" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-07", "revision": { "id": "revision_qwen_revision_qwen_20240807_introducing_qwen2_math_qwen2_math", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2-math" }, { "id": "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-08-28", "revision": { "id": "revision_qwen_revision_qwen_20240828_qwen2_vl_to_see_the_world_more_clearly_qwen2_vl", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2-vl" }, { "id": "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "revision": { "id": "revision_qwen_revision_qwen_20240918_qwen2_5_a_party_of_foundation_models_qwen2_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5" }, { "id": "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "revision": { "id": "revision_qwen_revision_qwen_20240918_qwen2_5_coder_code_more_learn_more_qwen2_5_coder", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-coder" }, { "id": "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "revision": { "id": "revision_qwen_revision_qwen_20240918_qwen2_5_llm_extending_the_boundary_of_llms_qwen2_5_llm", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-llm" }, { "id": "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-09-18", "revision": { "id": "revision_qwen_revision_qwen_20240918_qwen2_5_math_the_world_s_leading_open_sourced_mathem_qwen2_5_math", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-math" }, { "id": "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-11", "revision": { "id": "revision_qwen_revision_qwen_20241111_qwen2_5_coder_series_powerful_diverse_practical_qwen2_5_coder_family", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-coder-family" }, { "id": "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-14", "revision": { "id": "revision_qwen_revision_qwen_20241114_extending_the_context_length_to_1m_tokens_qwen2_5_turbo", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-turbo" }, { "id": "source_qwen_source_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-11-27", "revision": { "id": "revision_qwen_revision_qwen_20241127_qwq_reflect_deeply_on_the_boundaries_of_the_unknown_qwq_32b_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwq-32b-preview" }, { "id": "source_qwen_source_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2024-12-24", "revision": { "id": "revision_qwen_revision_qwen_20241224_qvq_to_see_the_world_with_wisdom_qvq_72b_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qvq-72b-preview" }, { "id": "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "revision": { "id": "revision_qwen_revision_qwen_20250126_qwen2_5_1m_deploy_your_own_qwen_with_context_length_qwen2_5_1m", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-1m" }, { "id": "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-26", "revision": { "id": "revision_qwen_revision_qwen_20250126_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl_qwen2_5_vl", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-vl" }, { "id": "source_qwen_source_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-01-28", "revision": { "id": "revision_qwen_revision_qwen_20250128_qwen2_5_max_exploring_the_intelligence_of_large_scal_qwen2_5_max", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-max" }, { "id": "source_qwen_source_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-02-24", "revision": { "id": "revision_qwen_revision_qwen_20250224_think_think_qwq_max_preview_qwq_max_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwq-max-preview" }, { "id": "source_qwen_source_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-05", "revision": { "id": "revision_qwen_revision_qwen_20250305_qwq_32b_embracing_the_power_of_reinforcement_learnin_qwq_32b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwq-32b" }, { "id": "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-23", "revision": { "id": "revision_qwen_revision_qwen_20250323_qwen2_5_vl_32b_smarter_and_lighter_qwen2_5_vl_32b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-vl-32b" }, { "id": "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-26", "revision": { "id": "revision_qwen_revision_qwen_20250326_qwen2_5_omni_see_hear_talk_write_do_it_all_qwen2_5_omni", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen2.5-omni" }, { "id": "source_qwen_source_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-03-27", "revision": { "id": "revision_qwen_revision_qwen_20250327_qvq_max_think_with_evidence_qvq_max_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qvq-max-preview" }, { "id": "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250428_qwen3_think_deeper_act_faster_qwen3" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-04-28", "revision": { "id": "revision_qwen_revision_qwen_20250428_qwen3_think_deeper_act_faster_qwen3", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3" }, { "id": "source_qwen_source_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-06-26", "revision": { "id": "revision_qwen_revision_qwen_20250626_qwen_vlo_from_understanding_the_world_to_depicting_i_qwen_vlo", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen-vlo" }, { "id": "source_qwen_source_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-07-22", "revision": { "id": "revision_qwen_revision_qwen_20250722_qwen3_coder_agentic_coding_in_the_world_qwen3_coder", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-coder" }, { "id": "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-10", "revision": { "id": "revision_qwen_revision_qwen_20250910_qwen3_next_towards_ultimate_training_inference_effic_qwen3_next", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-next" }, { "id": "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-21", "revision": { "id": "revision_qwen_revision_qwen_20250921_qwen3_omni_natively_omni_modal_foundation_models_qwen3_omni", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-omni" }, { "id": "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-22", "revision": { "id": "revision_qwen_revision_qwen_20250922_qwen3_vl_sharper_vision_deeper_thought_broader_actio_qwen3_vl", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-vl" }, { "id": "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20250924_qwen3_max_just_scale_it_qwen3_max" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-09-24", "revision": { "id": "revision_qwen_revision_qwen_20250924_qwen3_max_just_scale_it_qwen3_max", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-max" }, { "id": "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2025-12-08", "revision": { "id": "revision_qwen_revision_qwen_20251208_qwen3_omni_flash_2025_12_01_hear_you_see_you_follow_qwen3_omni_flash_20251201", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-omni-flash-20251201" }, { "id": "source_qwen_source_qwen_20260126_qwen3_max_thinking", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260126_qwen3_max_thinking" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-01-26", "revision": { "id": "revision_qwen_revision_qwen_20260126_qwen3_max_thinking", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-max-thinking" }, { "id": "source_qwen_source_qwen_20260203_qwen3_coder_next", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260203_qwen3_coder_next" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-03", "revision": { "id": "revision_qwen_revision_qwen_20260203_qwen3_coder_next", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3-coder-next" }, { "id": "source_qwen_source_qwen_20260216_qwen3_5", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260216_qwen3_5" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-02-16", "revision": { "id": "revision_qwen_revision_qwen_20260216_qwen3_5", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.5" }, { "id": "source_qwen_source_qwen_20260319_qwen3_5_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260319_qwen3_5_max_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-19", "revision": { "id": "revision_qwen_revision_qwen_20260319_qwen3_5_max_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.5-max-preview" }, { "id": "source_qwen_source_qwen_20260330_qwen3_5_omni", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260330_qwen3_5_omni" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-03-30", "revision": { "id": "revision_qwen_revision_qwen_20260330_qwen3_5_omni", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.5-omni" }, { "id": "source_qwen_source_qwen_20260402_qwen3_6", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260402_qwen3_6" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-02", "revision": { "id": "revision_qwen_revision_qwen_20260402_qwen3_6", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.6" }, { "id": "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260415_qwen3_6_35b_a3b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-15", "revision": { "id": "revision_qwen_revision_qwen_20260415_qwen3_6_35b_a3b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.6-35b-a3b" }, { "id": "source_qwen_source_qwen_20260418_qwen3_6_max_preview", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260418_qwen3_6_max_preview" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-18", "revision": { "id": "revision_qwen_revision_qwen_20260418_qwen3_6_max_preview", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.6-max-preview" }, { "id": "source_qwen_source_qwen_20260422_qwen3_6_27b", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260422_qwen3_6_27b" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-04-22", "revision": { "id": "revision_qwen_revision_qwen_20260422_qwen3_6_27b", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.6-27b" }, { "id": "source_qwen_source_qwen_20260520_qwen3_7", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260520_qwen3_7" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-05-20", "revision": { "id": "revision_qwen_revision_qwen_20260520_qwen3_7", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.7" }, { "id": "source_qwen_source_qwen_20260601_qwen3_7_plus", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260601_qwen3_7_plus" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-06-01", "revision": { "id": "revision_qwen_revision_qwen_20260601_qwen3_7_plus", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.7-plus" }, { "id": "source_qwen_source_qwen_20260803_qwen3_8", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260803_qwen3_8" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-03", "revision": { "id": "revision_qwen_revision_qwen_20260803_qwen3_8", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.8" }, { "id": "source_qwen_source_qwen_20260826_qwen3_8_flash_next", "lab_id": "qwen", "provenance": { "record_ids": [ "source_qwen_source_qwen_20260826_qwen3_8_flash_next" ], "record_type": "sources", "source_file": "data/catalog.yaml" }, "publication_date": "2026-08-26", "revision": { "id": "revision_qwen_revision_qwen_20260826_qwen3_8_flash_next", "retrieval_date": "2026-09-01", "unavailable_reason": "mutable_web_page_no_stable_revision" }, "source_type": "launch_page", "url": "https://qwen.ai/research/qwen3.8-flash-next" } ] }